diff --git a/.gitattributes b/.gitattributes index dd9b1d2e9866842b0f798cf2763ad8371e6b5602..a6e67b5b1883bf82d2cf2c95869c76090930d7b9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -37,3 +37,4 @@ wandb/run-20240408_203054-ldiuneeg/run-ldiuneeg.wandb filter=lfs diff=lfs merge= wandb/run-20240410_010250-ft4a6i5j/run-ft4a6i5j.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20240410_115017-r9p6sok2/run-r9p6sok2.wandb filter=lfs diff=lfs merge=lfs -text tokenizer.json filter=lfs diff=lfs merge=lfs -text +wandb/run-20240411_040722-rimyh9a1/run-rimyh9a1.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index e8a990c57de11f1a34f500e8df7757765bd41444..8b72e73f3534d7483b11506d5032b28c82d99005 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The following hyperparameters were used during training: - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08 - lr_scheduler_type: cosine - lr_scheduler_warmup_ratio: 0.03 -- training_steps: 10 +- training_steps: 20000 ### Training results diff --git a/adapter_config.json b/adapter_config.json index a585500adc599faecfa4747bbb26fe35a436d289..931f768c87269f55227eb55da9c811722a09a972 100644 --- a/adapter_config.json +++ b/adapter_config.json @@ -20,8 +20,8 @@ "rank_pattern": {}, "revision": null, "target_modules": [ - "lm_head", "query_key_value", + "lm_head", "dense_h_to_4h" ], "task_type": "CAUSAL_LM", diff --git a/adapter_model.safetensors b/adapter_model.safetensors index 71e018da716d2bc0b647b65a85eb6d29aa888f26..303292c48d0c7ba6981a630300b7ab0f107a1125 100644 --- a/adapter_model.safetensors +++ b/adapter_model.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9baf0d7e0bee813113ef40ed93ff8433a426745af90937aabc260a737c8f5277 +oid sha256:7ac730ac4465d0ec9fbe49ed9c5ad2a69ec11ddf9dc1d9cb0f5595f60817af39 size 1042757704 diff --git a/bloomz_dam_my/README.md b/bloomz_dam_my/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cf548c8a306e2af4c58a8e8ad5a2b6d6e4edcc98 --- /dev/null +++ b/bloomz_dam_my/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: bigscience/bloomz-560m +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.10.0 \ No newline at end of file diff --git a/bloomz_dam_my/adapter_config.json b/bloomz_dam_my/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a585500adc599faecfa4747bbb26fe35a436d289 --- /dev/null +++ b/bloomz_dam_my/adapter_config.json @@ -0,0 +1,30 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "bigscience/bloomz-560m", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.01, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "lm_head", + "query_key_value", + "dense_h_to_4h" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/bloomz_dam_my/adapter_model.safetensors b/bloomz_dam_my/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..71e018da716d2bc0b647b65a85eb6d29aa888f26 --- /dev/null +++ b/bloomz_dam_my/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9baf0d7e0bee813113ef40ed93ff8433a426745af90937aabc260a737c8f5277 +size 1042757704 diff --git a/bloomz_dam_my/special_tokens_map.json b/bloomz_dam_my/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..d8cbbd1c17c59f60ffb421543c870e5ac8665f52 --- /dev/null +++ b/bloomz_dam_my/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/bloomz_dam_my/spiece.model b/bloomz_dam_my/spiece.model new file mode 100644 index 0000000000000000000000000000000000000000..e417801865fd66bd40f9d45d46b6d0d0c2aa36b6 --- /dev/null +++ b/bloomz_dam_my/spiece.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6 +size 4309802 diff --git a/bloomz_dam_my/tokenizer.json b/bloomz_dam_my/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1e7fc7117b7be36863e0037375394dc9a7cc4b12 --- /dev/null +++ b/bloomz_dam_my/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8af11292e1e8b6e7fc393100a39b873778106b40afa8ef4e7c198f0bb5c03f82 +size 16315312 diff --git a/bloomz_dam_my/tokenizer_config.json b/bloomz_dam_my/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..322338561463d2a97738008a11ac9fb61e68d083 --- /dev/null +++ b/bloomz_dam_my/tokenizer_config.json @@ -0,0 +1,39 @@ +{ + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [], + "clean_up_tokenization_spaces": true, + "eos_token": "", + "extra_ids": 0, + "max_length": 1024, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "tokenizer_class": "T5Tokenizer", + "truncation": true, + "unk_token": "" +} diff --git a/bloomz_dam_my/training_args.bin b/bloomz_dam_my/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..5f35bd04ba64c14e64b849c815ed1740f3479fa2 --- /dev/null +++ b/bloomz_dam_my/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fcb5d40d27446b473f3cf6a16217747341cc71590de85e4a2a9001a3be0bfca +size 4920 diff --git a/runs/Apr11_04-02-02_321ddc84c049/events.out.tfevents.1712808126.321ddc84c049.414.0 b/runs/Apr11_04-02-02_321ddc84c049/events.out.tfevents.1712808126.321ddc84c049.414.0 new file mode 100644 index 0000000000000000000000000000000000000000..0da806fb8ff25a22fa5cd646b492f8971840566b --- /dev/null +++ b/runs/Apr11_04-02-02_321ddc84c049/events.out.tfevents.1712808126.321ddc84c049.414.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc0de91e150ffa3ae50d49ef29cc00f0d6cd90e2dfce554c500adb81011433e +size 15910 diff --git a/runs/Apr11_04-04-49_321ddc84c049/events.out.tfevents.1712808294.321ddc84c049.740.0 b/runs/Apr11_04-04-49_321ddc84c049/events.out.tfevents.1712808294.321ddc84c049.740.0 new file mode 100644 index 0000000000000000000000000000000000000000..9886dc3bf62a3edc9487be34a3e4ad45d2aa1b0d --- /dev/null +++ b/runs/Apr11_04-04-49_321ddc84c049/events.out.tfevents.1712808294.321ddc84c049.740.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77ccd6bb0384d7f0b531b4689a6d5091c5cf8ef5fe830dd33a06449fc0a4e6c6 +size 15706 diff --git a/runs/Apr11_04-07-17_321ddc84c049/events.out.tfevents.1712808441.321ddc84c049.1050.0 b/runs/Apr11_04-07-17_321ddc84c049/events.out.tfevents.1712808441.321ddc84c049.1050.0 new file mode 100644 index 0000000000000000000000000000000000000000..fa2d2e4740e50ac037975925bb046bdff1b94fa4 --- /dev/null +++ b/runs/Apr11_04-07-17_321ddc84c049/events.out.tfevents.1712808441.321ddc84c049.1050.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39731fc62d56057a7ac2a4a59ab5f57cc8eafb01f7eadaa3b6146959084621fe +size 233938 diff --git a/tokenizer.json b/tokenizer.json index 1e7fc7117b7be36863e0037375394dc9a7cc4b12..6f07fa0167afe486178d0fbf0725299bcd27c9a4 100644 --- a/tokenizer.json +++ b/tokenizer.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8af11292e1e8b6e7fc393100a39b873778106b40afa8ef4e7c198f0bb5c03f82 -size 16315312 +oid sha256:54e5c72a5ea09da48b2f316760b8bc5a445683ab9a5bc6b68db5d8db624ecceb +size 16315213 diff --git a/training_args.bin b/training_args.bin index 5f35bd04ba64c14e64b849c815ed1740f3479fa2..d7b57b454b35dce60603a238906785ddb4fd93d5 100644 --- a/training_args.bin +++ b/training_args.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9fcb5d40d27446b473f3cf6a16217747341cc71590de85e4a2a9001a3be0bfca +oid sha256:7ba44607bb84c5fd76d74e8d257eb1a18a8dceedf164c95e0e7b4eb4771e56e1 size 4920 diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log index 6d27e973415956c682f3d3f4931936c37532a9bc..a2d91a1066637700385639e51652426782b2958a 100644 --- a/wandb/debug-internal.log +++ b/wandb/debug-internal.log @@ -1,32 +1,32 @@ -2024-04-11 03:59:09,940 INFO StreamThr :191 [internal.py:wandb_internal():86] W&B internal server running at pid: 191, started at: 2024-04-11 03:59:09.939664 -2024-04-11 03:59:09,941 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status -2024-04-11 03:59:10,363 INFO WriterThread:191 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240411_035909-21esl9hl/run-21esl9hl.wandb -2024-04-11 03:59:10,364 DEBUG SenderThread:191 [sender.py:send():379] send: header -2024-04-11 03:59:10,367 DEBUG SenderThread:191 [sender.py:send():379] send: run -2024-04-11 03:59:10,517 INFO SenderThread:191 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files -2024-04-11 03:59:10,517 INFO SenderThread:191 [sender.py:_start_run_threads():1124] run started: 21esl9hl with start time 1712807949.940735 -2024-04-11 03:59:10,526 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: check_version -2024-04-11 03:59:10,526 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: check_version -2024-04-11 03:59:10,621 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: run_start -2024-04-11 03:59:10,635 DEBUG HandlerThread:191 [system_info.py:__init__():26] System info init -2024-04-11 03:59:10,635 DEBUG HandlerThread:191 [system_info.py:__init__():41] System info init done -2024-04-11 03:59:10,635 INFO HandlerThread:191 [system_monitor.py:start():194] Starting system monitor -2024-04-11 03:59:10,635 INFO SystemMonitor:191 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-11 03:59:10,635 INFO HandlerThread:191 [system_monitor.py:probe():214] Collecting system info -2024-04-11 03:59:10,639 INFO SystemMonitor:191 [interfaces.py:start():190] Started cpu monitoring -2024-04-11 03:59:10,640 INFO SystemMonitor:191 [interfaces.py:start():190] Started disk monitoring -2024-04-11 03:59:10,644 INFO SystemMonitor:191 [interfaces.py:start():190] Started gpu monitoring -2024-04-11 03:59:10,644 INFO SystemMonitor:191 [interfaces.py:start():190] Started memory monitoring -2024-04-11 03:59:10,645 INFO SystemMonitor:191 [interfaces.py:start():190] Started network monitoring -2024-04-11 03:59:10,650 DEBUG HandlerThread:191 [system_info.py:probe():150] Probing system -2024-04-11 03:59:10,652 DEBUG HandlerThread:191 [gitlib.py:_init_repo():56] git repository is invalid -2024-04-11 03:59:10,652 DEBUG HandlerThread:191 [system_info.py:probe():198] Probing system done -2024-04-11 03:59:10,652 DEBUG HandlerThread:191 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-11T03:59:10.650560', 'startedAt': '2024-04-11T03:59:09.933024', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '321ddc84c049', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.19, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5566.961940765381}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} -2024-04-11 03:59:10,653 INFO HandlerThread:191 [system_monitor.py:probe():224] Finished collecting system info -2024-04-11 03:59:10,653 INFO HandlerThread:191 [system_monitor.py:probe():227] Publishing system info -2024-04-11 03:59:10,653 DEBUG HandlerThread:191 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-04-11 03:59:11,520 INFO Thread-12 :191 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/conda-environment.yaml -2024-04-11 03:59:25,667 ERROR HandlerThread:191 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-04-11 04:07:22,850 INFO StreamThr :1091 [internal.py:wandb_internal():86] W&B internal server running at pid: 1091, started at: 2024-04-11 04:07:22.849871 +2024-04-11 04:07:22,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status +2024-04-11 04:07:23,106 INFO WriterThread:1091 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/run-rimyh9a1.wandb +2024-04-11 04:07:23,107 DEBUG SenderThread:1091 [sender.py:send():379] send: header +2024-04-11 04:07:23,109 DEBUG SenderThread:1091 [sender.py:send():379] send: run +2024-04-11 04:07:23,229 INFO SenderThread:1091 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files +2024-04-11 04:07:23,229 INFO SenderThread:1091 [sender.py:_start_run_threads():1124] run started: rimyh9a1 with start time 1712808442.848646 +2024-04-11 04:07:23,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: check_version +2024-04-11 04:07:23,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: check_version +2024-04-11 04:07:23,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: run_start +2024-04-11 04:07:23,341 DEBUG HandlerThread:1091 [system_info.py:__init__():26] System info init +2024-04-11 04:07:23,341 DEBUG HandlerThread:1091 [system_info.py:__init__():41] System info init done +2024-04-11 04:07:23,341 INFO HandlerThread:1091 [system_monitor.py:start():194] Starting system monitor +2024-04-11 04:07:23,341 INFO SystemMonitor:1091 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-11 04:07:23,342 INFO HandlerThread:1091 [system_monitor.py:probe():214] Collecting system info +2024-04-11 04:07:23,342 INFO SystemMonitor:1091 [interfaces.py:start():190] Started cpu monitoring +2024-04-11 04:07:23,343 INFO SystemMonitor:1091 [interfaces.py:start():190] Started disk monitoring +2024-04-11 04:07:23,344 INFO SystemMonitor:1091 [interfaces.py:start():190] Started gpu monitoring +2024-04-11 04:07:23,345 INFO SystemMonitor:1091 [interfaces.py:start():190] Started memory monitoring +2024-04-11 04:07:23,346 INFO SystemMonitor:1091 [interfaces.py:start():190] Started network monitoring +2024-04-11 04:07:23,361 DEBUG HandlerThread:1091 [system_info.py:probe():150] Probing system +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_info.py:probe():198] Probing system done +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-11T04:07:23.361534', 'startedAt': '2024-04-11T04:07:22.840819', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '321ddc84c049', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.19, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5566.962131500244}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-04-11 04:07:23,364 INFO HandlerThread:1091 [system_monitor.py:probe():224] Finished collecting system info +2024-04-11 04:07:23,364 INFO HandlerThread:1091 [system_monitor.py:probe():227] Publishing system info +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-11 04:07:24,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/conda-environment.yaml +2024-04-11 04:07:38,380 ERROR HandlerThread:1091 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda subprocess.call( @@ -37,82 +37,15313 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-04-11 03:59:25,670 DEBUG HandlerThread:191 [system_info.py:_save_conda():222] Saving conda packages done -2024-04-11 03:59:25,671 INFO HandlerThread:191 [system_monitor.py:probe():229] Finished publishing system info -2024-04-11 03:59:25,678 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:25,678 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: keepalive -2024-04-11 03:59:25,679 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:25,679 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: keepalive -2024-04-11 03:59:25,679 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:25,679 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: keepalive -2024-04-11 03:59:25,679 DEBUG SenderThread:191 [sender.py:send():379] send: files -2024-04-11 03:59:25,680 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now -2024-04-11 03:59:25,883 INFO wandb-upload_0:191 [upload_job.py:push():131] Uploaded file /tmp/tmpgmb8demxwandb/zh9coxtl-wandb-metadata.json -2024-04-11 03:59:26,522 INFO Thread-12 :191 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/wandb-metadata.json -2024-04-11 03:59:26,714 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: python_packages -2024-04-11 03:59:26,714 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: python_packages -2024-04-11 03:59:26,717 DEBUG SenderThread:191 [sender.py:send():379] send: telemetry -2024-04-11 03:59:26,727 DEBUG SenderThread:191 [sender.py:send():379] send: config -2024-04-11 03:59:26,728 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-11 03:59:26,729 DEBUG SenderThread:191 [sender.py:send():379] send: metric -2024-04-11 03:59:26,729 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status -2024-04-11 03:59:26,730 DEBUG SenderThread:191 [sender.py:send():379] send: telemetry -2024-04-11 03:59:26,731 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status -2024-04-11 03:59:26,776 DEBUG SenderThread:191 [sender.py:send():379] send: metric -2024-04-11 03:59:26,776 WARNING SenderThread:191 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-04-11 03:59:26,776 DEBUG SenderThread:191 [sender.py:send():379] send: telemetry -2024-04-11 03:59:27,522 INFO Thread-12 :191 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/requirements.txt -2024-04-11 03:59:27,523 INFO Thread-12 :191 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log -2024-04-11 03:59:29,523 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log -2024-04-11 03:59:30,951 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:31,524 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log -2024-04-11 03:59:33,525 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log -2024-04-11 03:59:35,952 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:40,958 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:41,528 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/config.yaml -2024-04-11 03:59:41,717 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-11 03:59:41,717 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status -2024-04-11 03:59:41,718 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status -2024-04-11 03:59:45,530 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log -2024-04-11 03:59:46,261 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:51,262 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:56,417 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 03:59:56,717 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status -2024-04-11 03:59:56,718 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-11 03:59:56,719 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status -2024-04-11 03:59:57,534 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log -2024-04-11 04:00:01,827 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 04:00:06,828 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 04:00:08,943 DEBUG SenderThread:191 [sender.py:send():379] send: telemetry -2024-04-11 04:00:08,983 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: partial_history -2024-04-11 04:00:08,985 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: summary_record -2024-04-11 04:00:08,988 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-11 04:00:08,988 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: summary_record -2024-04-11 04:00:08,989 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-11 04:00:08,989 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: summary_record -2024-04-11 04:00:08,989 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-11 04:00:08,989 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: summary_record -2024-04-11 04:00:08,990 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-11 04:00:08,990 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: summary_record -2024-04-11 04:00:08,990 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-11 04:00:08,990 DEBUG SenderThread:191 [sender.py:send():379] send: metric -2024-04-11 04:00:08,991 DEBUG SenderThread:191 [sender.py:send():379] send: history -2024-04-11 04:00:08,991 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: summary_record -2024-04-11 04:00:08,991 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-11 04:00:09,539 INFO Thread-12 :191 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json -2024-04-11 04:00:10,645 DEBUG SystemMonitor:191 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-11 04:00:10,647 DEBUG SenderThread:191 [sender.py:send():379] send: stats -2024-04-11 04:00:11,539 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log -2024-04-11 04:00:11,715 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status -2024-04-11 04:00:11,716 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status -2024-04-11 04:00:11,719 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-11 04:00:11,833 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 04:00:12,540 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/config.yaml -2024-04-11 04:00:16,925 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 04:00:21,926 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 04:00:26,716 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status -2024-04-11 04:00:26,716 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status -2024-04-11 04:00:26,757 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-11 04:00:27,786 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 04:00:32,787 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report -2024-04-11 04:00:37,788 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,381 DEBUG HandlerThread:1091 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-11 04:07:38,381 INFO HandlerThread:1091 [system_monitor.py:probe():229] Finished publishing system info +2024-04-11 04:07:38,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,388 DEBUG SenderThread:1091 [sender.py:send():379] send: files +2024-04-11 04:07:38,388 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-11 04:07:38,590 INFO wandb-upload_0:1091 [upload_job.py:push():131] Uploaded file /tmp/tmppwkfxjn1wandb/7mpz5val-wandb-metadata.json +2024-04-11 04:07:39,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-metadata.json +2024-04-11 04:07:39,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: python_packages +2024-04-11 04:07:39,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: python_packages +2024-04-11 04:07:39,319 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,329 DEBUG SenderThread:1091 [sender.py:send():379] send: config +2024-04-11 04:07:39,330 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:07:39,331 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,332 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:07:39,332 WARNING SenderThread:1091 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-11 04:07:39,332 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:07:39,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:07:39,335 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:07:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/requirements.txt +2024-04-11 04:07:42,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:43,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:44,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:46,236 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:48,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:53,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:54,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 04:07:55,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:07:55,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:07:55,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:07:59,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:04,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:09,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:09,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:09,452 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:09,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:15,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:20,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:23,346 DEBUG SystemMonitor:1091 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-11 04:08:23,348 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:08:24,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:24,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:24,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:26,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:28,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:08:28,054 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:08:28,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:08:30,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:08:31,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:34,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:08:36,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:39,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:39,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:39,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:42,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:47,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:52,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:53,349 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:08:54,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:54,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:54,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:58,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:59,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 04:09:03,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:08,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:09,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:09,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:09,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:12,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:09:12,735 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:09:12,735 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:09:12,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:09:13,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:09:13,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:14,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:18,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:19,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:23,350 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:09:24,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:24,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:24,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:24,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:30,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:35,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:39,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:39,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:39,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:41,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:09:41,116 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:09:41,116 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:09:41,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:41,119 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:09:41,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:09:44,285 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:46,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:46,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:51,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:53,351 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:09:54,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:54,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:54,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:57,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:02,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:07,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:09,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:09,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:09,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:10,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:10:10,747 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:10:10,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:10:10,750 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:10:11,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:10:12,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:12,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:16,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:17,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:22,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:23,352 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:10:24,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:24,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:24,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:28,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:33,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:38,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:39,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:39,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:39,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:40,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:10:40,953 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:10:40,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:10:40,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:10:41,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:10:42,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:43,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:46,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:49,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:53,353 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:10:54,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:54,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:54,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:55,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:00,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:05,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:09,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:09,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:09,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:11,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:15,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:11:15,551 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:11:15,551 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:11:15,553 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:11:16,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:11:16,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:18,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:20,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:22,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:22,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:23,354 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:11:24,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:24,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:24,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:28,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:33,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:38,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:39,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:39,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:39,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:40,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:44,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:51,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:11:51,548 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:11:51,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:11:51,549 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:11:52,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:11:53,354 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:11:54,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:54,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:54,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:54,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:54,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:56,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:00,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:05,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:05,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:05,936 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:05,937 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:05,937 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:06,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:08,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:09,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:09,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:09,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:10,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:11,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:16,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:21,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:23,355 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:12:24,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:24,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:24,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:27,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:31,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:31,049 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:31,049 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:31,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:31,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:32,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:33,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:36,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:38,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:39,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:39,486 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:39,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:44,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:47,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:47,842 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:47,842 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:47,842 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:48,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:50,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:52,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:53,356 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:12:54,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:54,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:54,491 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:55,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:00,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:05,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:07,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:07,619 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:07,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:07,619 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:08,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:10,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:10,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:10,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:10,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:10,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:12,368 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:15,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:20,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:21,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:21,256 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:21,256 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:21,258 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:21,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:22,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:23,358 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:13:25,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:25,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:25,210 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:26,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:26,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:31,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:36,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:39,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:39,561 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:39,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:39,562 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:40,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:40,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:40,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:40,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:42,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:42,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:44,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:47,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:52,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:53,358 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:13:55,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:55,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:55,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:55,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:55,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:55,394 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:55,395 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:56,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:58,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:58,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:00,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:03,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:08,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:10,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:10,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:14,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:15,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:15,976 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:15,977 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:15,977 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:16,399 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:18,400 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:20,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:20,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:23,360 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:14:25,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:25,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:25,213 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:25,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:29,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:29,755 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:29,756 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:29,756 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:30,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:30,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:32,406 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:34,407 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:36,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:40,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:41,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:46,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:49,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:49,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:49,456 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:49,456 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:50,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:51,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:52,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:53,360 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:14:54,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:55,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:55,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:55,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:57,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:02,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:05,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:05,935 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:05,936 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:05,936 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:06,418 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:07,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:08,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:10,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:10,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:13,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:18,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:22,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:22,440 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:22,441 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:22,441 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:23,361 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:15:23,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:23,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:24,425 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:25,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:25,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:25,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:26,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:28,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:28,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:33,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:38,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:40,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:40,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:40,339 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:40,339 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:40,340 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:40,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:42,432 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:44,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:44,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:49,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:53,362 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:15:55,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:55,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:55,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:55,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:57,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:57,763 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:57,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:57,764 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:58,438 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:00,439 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:00,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:02,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:06,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:10,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:10,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:11,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:16,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:16,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:16,568 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:16,568 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:16,568 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:17,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:18,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:21,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:22,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:23,363 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:16:25,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:25,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:25,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:27,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:32,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:32,171 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:32,171 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:32,172 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:32,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:33,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:34,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:36,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:38,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:40,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:40,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:40,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:44,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:49,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:53,364 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:16:54,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:54,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:54,714 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:54,714 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:54,715 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:55,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:55,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:55,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:55,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:56,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:59,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:00,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:04,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:09,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:10,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:10,219 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:10,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:11,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:11,031 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:11,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:11,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:11,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:12,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:15,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:16,468 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:20,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:23,365 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:17:25,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:25,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:25,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:25,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:28,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:28,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:28,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:28,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:28,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:30,473 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:31,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:34,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:37,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:40,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:40,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:42,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:46,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:46,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:46,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:46,044 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:46,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:48,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:48,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:50,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:53,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:53,367 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:17:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:55,219 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:58,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:03,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:05,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:05,870 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:05,870 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:05,871 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:06,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:08,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:08,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:10,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:10,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:10,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:10,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:14,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:19,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:23,367 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:18:24,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:24,895 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:24,896 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:24,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:24,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:25,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:25,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:25,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:25,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:26,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:30,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:30,496 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:35,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:40,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:40,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:40,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:45,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:46,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:46,066 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:46,066 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:46,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:46,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:48,503 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:50,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:50,504 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:53,368 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:18:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:55,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:55,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:56,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:01,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:04,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:04,875 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:04,875 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:04,876 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:05,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:06,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:06,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:10,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:10,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:10,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:10,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:12,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:17,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:22,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:22,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:22,759 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:22,759 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:22,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:23,369 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:19:23,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:24,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:25,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:25,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:25,352 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:26,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:27,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:28,518 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:32,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:37,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:39,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:39,841 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:39,841 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:39,842 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:40,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:40,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:40,350 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:40,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:40,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:42,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:44,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:47,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:52,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:53,370 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:19:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:55,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:58,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:00,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:00,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:00,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:00,025 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:00,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:02,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:04,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:04,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:09,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:10,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:10,350 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:10,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:14,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:19,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:19,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:19,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:19,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:19,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:20,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:20,537 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:23,371 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:20:24,538 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:25,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:25,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:25,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:25,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:30,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:35,422 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:36,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:36,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:36,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:36,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:36,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:38,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:40,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:40,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:40,480 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:40,544 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:45,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:50,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:52,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:52,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:52,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:52,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:52,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:53,372 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:20:54,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:55,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:55,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:55,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:55,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:56,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:00,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:05,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:10,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:10,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:10,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:10,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:11,000 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:11,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:11,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:11,001 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:11,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:12,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:14,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:16,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:16,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:21,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:23,373 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:21:25,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:25,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:25,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:26,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:31,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:32,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:32,406 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:32,406 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:32,406 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:32,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:34,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:36,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:36,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:40,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:40,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:40,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:42,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:47,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:52,419 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:53,374 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:21:55,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:55,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:55,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:57,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:57,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:57,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:57,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:57,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:58,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:58,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:02,580 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:03,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:08,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:10,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:10,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:10,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:13,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:18,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:21,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:21,794 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:21,794 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:21,795 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:22,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:22,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:23,375 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:22:24,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:25,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:25,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:25,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:26,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:29,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:34,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:36,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:36,442 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:36,442 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:36,443 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:36,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:38,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:39,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:40,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:40,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:40,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:40,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:45,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:50,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:53,376 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:22:55,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:55,088 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:55,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:55,088 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:55,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:55,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:55,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:55,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:55,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:56,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:58,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:00,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:01,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:06,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:10,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:10,357 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:10,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:11,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:16,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:18,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:18,859 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:18,859 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:18,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:19,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:20,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:21,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:22,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:23,377 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:23:25,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:25,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:25,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:27,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:32,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:35,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:35,184 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:35,185 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:35,185 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:35,614 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:36,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:38,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:38,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:40,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:40,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:40,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:40,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:43,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:48,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:52,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:52,722 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:52,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:52,723 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:53,378 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:23:53,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:54,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:54,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:55,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:55,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:56,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:59,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:04,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:09,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:10,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:10,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:13,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:13,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:13,435 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:13,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:13,642 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:15,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:19,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:20,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:23,380 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:24:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:25,358 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:25,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:26,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:31,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:31,269 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:31,269 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:31,269 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:31,648 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:32,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:33,649 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:35,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:37,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:37,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:40,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:40,358 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:40,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:42,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:47,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:48,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:48,190 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:48,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:48,190 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:48,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:49,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:53,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:53,380 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:24:53,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:55,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:55,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:58,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:03,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:04,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:04,099 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:04,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:04,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:04,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:05,661 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:09,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:09,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:10,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:10,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:10,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:14,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:19,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:19,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:19,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:19,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:19,822 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:20,666 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:21,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:23,381 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:25:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:25,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:25,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:25,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:25,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:30,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:35,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:40,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:40,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:40,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:40,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:40,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:40,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:40,669 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:40,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:43,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:45,676 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:45,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:50,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:53,382 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:25:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:55,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:55,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:56,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:56,384 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:56,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:56,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:56,386 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:56,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:57,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:01,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:01,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:06,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:10,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:10,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:11,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:11,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:11,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:11,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:11,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:12,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:13,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:16,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:17,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:21,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:23,383 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:26:25,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:25,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:25,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:26,636 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:26,637 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:26,638 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:26,638 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:26,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:27,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:29,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:31,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:32,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:37,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:40,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:40,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:43,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:48,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:49,130 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:49,132 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:49,132 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:49,132 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:49,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:51,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:53,384 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:26:53,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:54,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:55,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:55,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:55,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:59,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:04,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:05,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:05,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:05,055 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:05,055 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:05,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:07,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:09,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:10,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:10,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:10,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:15,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:20,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:20,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:20,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:20,616 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:20,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:20,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:23,386 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:27:23,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:25,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:25,363 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:25,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:25,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:26,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:31,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:36,416 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:39,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:39,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:39,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:39,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:39,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:40,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:40,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:40,949 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:41,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:42,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:43,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:47,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:52,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:54,276 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:27:55,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:55,948 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:55,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:59,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:59,294 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:59,294 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:59,294 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:59,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:01,726 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:03,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:03,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:08,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:11,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:11,126 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:11,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:14,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:16,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:16,788 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:16,788 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:16,789 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:17,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:19,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:19,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:21,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:23,388 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:28:25,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:26,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:26,078 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:26,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:31,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:33,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:33,817 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:33,818 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:33,819 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:34,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:35,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:36,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:39,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:41,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:41,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:41,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:42,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:47,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:52,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:53,388 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:28:54,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:54,538 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:54,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:54,539 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:54,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:55,747 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:56,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:56,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:56,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:57,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:59,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:02,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:07,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:11,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:11,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:11,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:13,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:13,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:13,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:13,455 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:13,457 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:13,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:15,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:17,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:18,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:23,390 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:29:24,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:26,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:29,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:31,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:31,362 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:31,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:31,362 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:31,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:33,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:34,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:35,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:37,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:39,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:41,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:41,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:41,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:45,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:50,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:53,391 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:29:55,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:56,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:56,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:56,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:56,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:56,404 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:56,405 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:56,405 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:56,773 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:57,773 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:00,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:01,775 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:05,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:10,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:11,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:11,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:11,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:16,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:18,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:18,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:18,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:18,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:18,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:19,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:21,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:23,392 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:30:23,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:26,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:26,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:27,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:32,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:37,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:37,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:37,742 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:37,742 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:37,742 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:37,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:39,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:41,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:41,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:41,280 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:41,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:42,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:47,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:52,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:53,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:53,327 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:53,327 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:53,328 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:53,393 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:30:53,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:55,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:56,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:56,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:56,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:57,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:57,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:02,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:07,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:09,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:09,124 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:09,124 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:09,125 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:09,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:11,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:11,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:11,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:11,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:13,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:13,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:18,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:23,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:23,394 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:31:26,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:26,747 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:26,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:28,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:30,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:30,670 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:30,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:30,671 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:30,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:31,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:34,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:35,811 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:39,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:41,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:41,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:41,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:45,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:48,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:48,681 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:48,682 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:48,682 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:48,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:49,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:51,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:53,395 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:31:53,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:56,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:56,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:56,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:56,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:01,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:03,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:03,003 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:03,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:03,004 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:03,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:05,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:07,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:07,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:11,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:11,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:11,750 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:12,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:17,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:20,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:20,690 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:20,690 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:20,691 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:20,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:21,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:23,396 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:32:23,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:25,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:26,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:26,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:26,747 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:28,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:33,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:38,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:41,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:41,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:41,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:42,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:42,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:42,485 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:42,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:42,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:43,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:44,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:47,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:49,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:53,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:32:55,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:56,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:56,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:00,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:01,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:01,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:01,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:01,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:01,845 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:03,846 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:05,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:06,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:11,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:11,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:16,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:19,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:19,151 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:19,152 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:19,152 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:19,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:21,853 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:22,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:23,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:33:23,854 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:26,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:26,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:26,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:27,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:32,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:36,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:36,029 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:36,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:36,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:36,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:37,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:38,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:39,861 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:41,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:43,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:48,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:53,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:33:54,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:56,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:56,467 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:56,467 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:56,467 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:56,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:56,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:56,867 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:57,868 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:59,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:01,869 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:04,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:09,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:11,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:11,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:14,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:16,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:16,244 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:16,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:16,246 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:16,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:17,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:20,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:21,876 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:23,403 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:34:25,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:26,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:26,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:26,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:30,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:32,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:32,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:32,979 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:32,979 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:33,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:33,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:35,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:37,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:41,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:41,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:46,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:51,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:53,405 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:34:55,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:55,216 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:55,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:55,217 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:55,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:55,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:56,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:56,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:56,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:56,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:59,891 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:02,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:07,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:11,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:11,936 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:11,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:13,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:13,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:13,231 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:13,232 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:13,232 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:13,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:13,897 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:17,898 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:18,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:23,408 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:35:24,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:26,962 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:29,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:34,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:37,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:37,361 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:37,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:37,362 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:37,905 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:39,906 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:40,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:41,906 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:41,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:41,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:41,941 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:46,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:51,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:53,409 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:35:56,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:56,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:56,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:56,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:58,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:58,531 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:58,531 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:58,532 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:58,913 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:59,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:01,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:03,916 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:06,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:11,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:11,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:11,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:11,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:15,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:15,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:15,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:15,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:15,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:15,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:17,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:19,922 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:22,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:23,410 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:36:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:26,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:26,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:28,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:33,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:35,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:35,940 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:35,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:35,941 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:36,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:37,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:38,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:39,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:41,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:41,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:44,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:49,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:53,411 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:36:54,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:54,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:54,674 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:54,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:54,675 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:54,935 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:55,936 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:57,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:57,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:57,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:59,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:59,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:04,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:09,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:12,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:12,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:15,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:17,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:17,851 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:17,851 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:17,852 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:17,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:19,947 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:20,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:21,948 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:23,411 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:37:26,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:27,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:27,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:32,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:32,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:32,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:32,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:32,219 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:32,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:33,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:35,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:37,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:42,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:42,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:42,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:47,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:49,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:49,329 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:49,329 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:49,330 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:49,959 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:49,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:52,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:53,417 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:37:53,961 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:57,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:58,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:03,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:08,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:08,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:08,410 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:08,410 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:08,410 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:08,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:09,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:11,968 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:12,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:12,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:12,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:13,969 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:14,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:19,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:22,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:22,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:22,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:22,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:23,413 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:38:23,973 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:24,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:25,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:27,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:27,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:27,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:30,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:35,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:38,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:38,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:38,046 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:38,047 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:38,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:39,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:41,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:41,980 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:42,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:42,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:46,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:51,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:53,414 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:38:56,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:57,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:57,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:59,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:59,577 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:59,578 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:59,580 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:59,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:59,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:01,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:03,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:06,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:11,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:12,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:12,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:14,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:14,583 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:14,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:14,583 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:14,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:15,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:17,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:19,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:22,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:23,415 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:39:27,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:27,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:28,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:33,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:37,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:37,112 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:37,112 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:37,115 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:38,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:38,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:39,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:42,002 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:42,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:42,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:42,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:44,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:49,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:53,416 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:39:53,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:53,764 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:53,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:53,765 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:54,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:54,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:56,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:57,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:57,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:57,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:58,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:00,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:05,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:10,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:11,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:11,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:11,777 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:11,779 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:12,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:12,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:12,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:12,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:14,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:15,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:16,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:20,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:23,417 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:40:26,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:27,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:27,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:31,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:33,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:33,480 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:33,480 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:33,481 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:34,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:34,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:36,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:38,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:41,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:42,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:42,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:47,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:50,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:50,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:50,443 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:50,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:51,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:52,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:52,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:53,418 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:40:54,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:57,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:57,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:57,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:58,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:03,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:07,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:07,917 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:07,917 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:07,917 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:08,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:08,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:10,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:12,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:12,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:12,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:14,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:19,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:23,419 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:41:24,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:27,280 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:27,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:27,791 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:27,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:27,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:28,043 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:29,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:33,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:34,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:39,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:42,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:42,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:42,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:45,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:47,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:47,093 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:47,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:47,094 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:48,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:49,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:51,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:53,054 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:53,419 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:41:56,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:57,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:57,279 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:02,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:02,014 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:02,014 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:02,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:02,015 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:02,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:05,058 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:07,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:07,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:12,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:12,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:12,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:17,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:17,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:17,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:17,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:17,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:18,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:19,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:22,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:23,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:23,420 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:42:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:27,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:27,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:28,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:33,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:36,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:36,110 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:36,110 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:36,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:37,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:39,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:39,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:41,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:42,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:42,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:42,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:44,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:49,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:52,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:54,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:55,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:55,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:55,510 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:55,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:42:56,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:57,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:57,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:59,079 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:59,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:04,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:09,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:12,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:12,283 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:12,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:12,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:12,825 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:12,826 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:12,826 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:13,085 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:14,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:15,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:17,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:19,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:20,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:23,422 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:43:25,423 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:27,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:30,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:30,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:30,047 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:30,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:30,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:31,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:33,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:35,094 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:36,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:41,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:42,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:42,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:42,285 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:46,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:51,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:52,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:52,276 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:52,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:52,277 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:53,100 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:53,422 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:43:55,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:56,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:57,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:57,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:57,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:57,287 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:02,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:07,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:08,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:08,707 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:08,707 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:08,710 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:09,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:11,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:12,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:12,287 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:12,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:12,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:13,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:17,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:22,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:23,423 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:44:25,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:25,204 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:25,204 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:25,204 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:26,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:27,113 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:27,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:27,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:27,807 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:27,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:31,114 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:33,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:38,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:42,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:42,671 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:42,671 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:42,673 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:42,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:42,805 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:42,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:43,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:43,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:45,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:47,120 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:49,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:53,424 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:44:55,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:57,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:57,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:57,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:00,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:02,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:02,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:02,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:02,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:03,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:06,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:07,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:11,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:12,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:12,807 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:12,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:16,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:21,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:23,425 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:45:25,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:25,457 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:25,457 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:25,458 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:26,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:27,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:27,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:28,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:28,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:28,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:31,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:32,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:37,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:42,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:43,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:43,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:43,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:46,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:46,088 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:46,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:46,089 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:46,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:47,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:48,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:51,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:53,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:53,426 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:45:58,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:58,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:58,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:59,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:04,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:09,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:10,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:10,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:10,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:10,992 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:11,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:13,152 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:13,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:15,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:15,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:20,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:23,427 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:46:25,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:25,188 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:25,188 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:25,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:25,189 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:26,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:27,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:28,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:28,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:28,592 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:29,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:30,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:31,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:35,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:40,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:43,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:43,585 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:44,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:44,209 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:44,209 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:44,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:45,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:45,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:46,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:49,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:51,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:53,428 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:46:56,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:58,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:58,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:58,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:00,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:00,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:00,428 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:00,428 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:01,170 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:01,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:03,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:05,172 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:06,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:11,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:13,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:13,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:13,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:16,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:17,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:17,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:17,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:17,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:18,176 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:19,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:21,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:23,178 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:23,429 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:47:27,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:28,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:28,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:32,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:35,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:35,720 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:35,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:35,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:36,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:37,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:37,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:41,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:43,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:43,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:43,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:48,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:53,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:55,387 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:47:58,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:58,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:58,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:59,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:00,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:00,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:00,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:00,242 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:01,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:01,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:05,194 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:05,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:10,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:13,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:13,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:13,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:15,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:17,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:17,898 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:17,899 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:17,899 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:18,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:19,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:20,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:23,201 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:23,430 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:48:26,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:28,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:28,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:31,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:31,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:31,943 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:31,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:31,945 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:32,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:33,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:37,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:37,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:42,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:43,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:43,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:45,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:45,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:45,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:46,209 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:47,209 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:47,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:51,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:52,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:53,431 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:48:58,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:58,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:58,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:58,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:03,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:08,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:08,340 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:08,340 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:08,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:09,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:09,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:09,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:13,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:13,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:14,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:19,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:23,432 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:49:25,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:28,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:28,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:28,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:28,803 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:28,803 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:28,804 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:29,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:30,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:31,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:33,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:35,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:40,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:43,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:43,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:45,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:45,351 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:45,352 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:45,354 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:46,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:46,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:47,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:49,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:51,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:53,433 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:49:57,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:58,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:58,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:58,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:02,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:05,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:05,630 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:05,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:05,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:06,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:07,240 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:07,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:11,241 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:12,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:13,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:18,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:23,434 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:50:24,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:27,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:27,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:27,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:27,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:28,247 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:29,248 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:29,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:29,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:29,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:30,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:31,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:33,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:35,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:40,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:44,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:46,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:47,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:47,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:47,778 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:47,778 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:48,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:49,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:51,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:53,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:53,434 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:50:57,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:59,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:59,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:59,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:03,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:08,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:11,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:11,189 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:11,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:11,190 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:11,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:13,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:14,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:14,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:14,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:14,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:15,268 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:19,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:23,435 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:51:24,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:29,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:29,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:30,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:30,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:30,379 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:30,379 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:30,379 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:31,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:31,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:35,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:35,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:40,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:44,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:44,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:45,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:45,524 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:45,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:45,525 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:46,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:46,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:47,285 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:51,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:51,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:53,436 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:51:57,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:59,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:59,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:59,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:00,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:00,796 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:00,796 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:00,797 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:01,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:02,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:03,291 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:05,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:08,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:13,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:14,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:14,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:14,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:15,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:15,101 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:15,101 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:15,102 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:15,299 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:17,300 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:18,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:19,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:23,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:23,437 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:52:28,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:29,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:34,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:36,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:36,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:36,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:36,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:36,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:37,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:39,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:41,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:44,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:44,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:44,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:49,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:51,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:54,356 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:54,356 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:54,359 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:54,359 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:52:54,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:55,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:55,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:59,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:59,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:59,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:00,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:05,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:10,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:12,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:12,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:12,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:12,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:12,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:13,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:14,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:14,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:14,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:16,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:17,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:21,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:23,439 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:53:26,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:29,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:29,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:29,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:31,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:35,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:35,475 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:35,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:35,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:36,330 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:37,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:37,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:39,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:42,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:44,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:44,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:44,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:47,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:50,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:50,459 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:50,460 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:50,460 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:51,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:51,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:53,440 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:53:53,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:55,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:58,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:59,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:59,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:59,644 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:03,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:07,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:07,353 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:07,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:07,356 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:08,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:09,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:09,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:11,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:14,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:14,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:14,629 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:14,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:19,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:22,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:22,647 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:22,647 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:22,648 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:23,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:23,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:23,441 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:54:25,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:27,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:29,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:29,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:29,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:30,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:35,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:40,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:44,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:44,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:44,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:44,556 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:44,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:44,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:44,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:45,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:45,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:45,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:49,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:50,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:53,442 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:54:56,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:59,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:59,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:59,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:01,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:02,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:02,729 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:02,729 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:02,730 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:03,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:03,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:06,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:07,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:11,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:14,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:14,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:14,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:17,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:20,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:20,139 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:20,140 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:20,141 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:20,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:21,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:23,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:23,443 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:55:25,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:28,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:29,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:29,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:29,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:33,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:38,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:42,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:42,914 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:42,914 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:42,915 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:43,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:43,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:43,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:44,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:44,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:44,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:47,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:49,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:53,444 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:55:54,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:59,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:59,626 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:59,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:00,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:00,713 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:00,713 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:00,713 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:01,384 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:01,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:04,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:05,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:09,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:14,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:14,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:14,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:15,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:19,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:19,145 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:19,145 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:19,146 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:19,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:19,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:21,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:23,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:23,445 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:56:26,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:29,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:29,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:29,845 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:31,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:36,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:36,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:36,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:36,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:36,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:37,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:37,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:41,399 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:42,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:44,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:44,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:44,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:47,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:52,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:53,446 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:56:56,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:56,842 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:56,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:56,843 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:57,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:57,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:58,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:59,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:59,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:59,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:01,406 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:03,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:08,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:13,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:14,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:14,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:14,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:19,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:21,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:57:21,105 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:57:21,106 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:57:21,106 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:57:21,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:57:21,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:23,447 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:57:25,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:25,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:29,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:29,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:29,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:30,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:35,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:38,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:57:38,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:57:38,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:57:38,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:57:38,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:57:39,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:41,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:41,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:44,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:44,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:44,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:46,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:51,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:53,452 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:57:57,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:59,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:59,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:59,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:01,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:01,265 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:01,265 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:01,265 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:01,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:01,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:03,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:05,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:08,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:13,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:14,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:14,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:14,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:18,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:18,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:18,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:18,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:18,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:19,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:19,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:23,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:23,449 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:58:24,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:29,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:29,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:35,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:35,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:35,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:35,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:35,719 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:36,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:37,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:39,444 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:40,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:44,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:44,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:44,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:45,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:50,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:52,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:52,924 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:52,925 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:52,925 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:53,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:53,450 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:58:54,450 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:55,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:58,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:59,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:59,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:59,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:01,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:06,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:09,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:09,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:09,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:09,555 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:10,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:12,457 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:12,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:14,457 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:14,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:14,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:14,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:17,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:22,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:23,451 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:59:24,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:24,289 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:24,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:24,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:24,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:26,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:28,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:28,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:29,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:33,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:38,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:43,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:44,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:44,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:44,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:44,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:44,928 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:44,928 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:44,931 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:45,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:46,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:49,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:50,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:53,452 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:59:54,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:59,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:59,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:59,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:59,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:03,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:03,227 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:03,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:03,227 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:03,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:04,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:05,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:08,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:10,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:14,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:14,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:14,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:15,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:20,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:23,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:23,045 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:23,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:23,048 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:23,453 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:00:23,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:24,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:26,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:28,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:29,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:31,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:36,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:41,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:43,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:43,247 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:43,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:43,248 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:43,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:44,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:44,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:47,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:48,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:52,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:53,455 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:00:57,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:59,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:59,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:59,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:00,640 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:00,640 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:00,641 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:01,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:02,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:02,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:06,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:07,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:12,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:14,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:17,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:17,328 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:17,328 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:17,329 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:18,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:18,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:22,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:23,456 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:01:23,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:28,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:29,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:33,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:34,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:34,859 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:34,859 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:34,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:35,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:36,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:39,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:40,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:44,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:44,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:45,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:50,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:53,457 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:01:55,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:59,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:59,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:59,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:01,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:01,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:01,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:01,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:01,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:02,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:04,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:06,525 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:06,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:11,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:14,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:14,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:17,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:22,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:23,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:23,123 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:23,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:23,124 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:23,457 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:02:23,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:24,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:27,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:28,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:29,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:29,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:33,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:38,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:42,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:42,062 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:42,063 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:42,063 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:42,538 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:43,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:44,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:44,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:44,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:44,982 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:46,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:48,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:53,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:53,459 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:02:58,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:59,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:59,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:59,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:02,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:02,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:02,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:02,180 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:02,546 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:04,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:04,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:06,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:08,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:09,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:14,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:14,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:14,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:14,968 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:19,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:20,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:20,561 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:20,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:20,562 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:21,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:22,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:23,460 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:03:25,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:26,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:29,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:29,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:29,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:31,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:36,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:38,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:38,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:38,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:38,291 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:38,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:40,560 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:41,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:42,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:44,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:44,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:44,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:44,996 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:47,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:52,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:53,460 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:03:57,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:58,059 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:58,059 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:58,060 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:58,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:00,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:00,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:00,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:00,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:03,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:08,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:13,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:15,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:15,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:15,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:15,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:15,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:15,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:16,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:18,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:20,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:23,461 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:04:23,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:28,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:30,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:30,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:33,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:38,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:41,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:41,062 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:41,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:41,063 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:41,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:42,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:44,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:45,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:45,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:45,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:46,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:49,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:53,462 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:04:55,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:55,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:55,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:55,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:55,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:56,589 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:58,590 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:00,591 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:00,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:00,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:00,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:00,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:05,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:10,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:15,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:15,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:15,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:15,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:17,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:05:17,261 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:05:17,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:05:17,261 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:05:17,597 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:05:20,598 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:21,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:22,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:23,463 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:05:27,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:30,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:30,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:32,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:37,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:41,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:05:41,340 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:05:41,340 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:05:41,341 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:05:41,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:05:42,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:43,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:45,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:45,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:45,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:46,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:48,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:53,464 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:05:54,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:59,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:00,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:00,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:00,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:00,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:00,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:00,656 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:02,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:05,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:06,619 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:10,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:15,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:15,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:15,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:15,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:20,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:22,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:22,657 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:22,657 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:22,657 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:23,465 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:06:23,625 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:24,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:25,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:28,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:30,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:30,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:36,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:39,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:39,451 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:39,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:39,452 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:39,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:40,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:42,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:44,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:45,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:45,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:45,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:47,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:52,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:53,467 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:06:57,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:57,310 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:57,310 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:57,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:57,639 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:58,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:58,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:00,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:00,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:00,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:02,648 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:03,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:08,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:13,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:15,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:15,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:15,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:16,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:16,886 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:16,886 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:16,886 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:17,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:18,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:18,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:22,658 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:23,467 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:07:24,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:29,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:30,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:30,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:30,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:34,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:39,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:40,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:40,780 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:40,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:40,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:41,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:42,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:44,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:45,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:45,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:45,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:46,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:50,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:53,469 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:07:56,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:57,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:57,405 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:57,406 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:57,406 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:57,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:58,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:00,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:00,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:01,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:02,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:06,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:11,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:12,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:12,997 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:12,997 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:12,997 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:13,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:14,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:15,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:15,713 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:15,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:17,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:18,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:22,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:23,469 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:08:27,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:29,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:29,908 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:29,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:29,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:30,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:30,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:32,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:32,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:34,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:38,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:43,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:45,659 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:45,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:48,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:48,006 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:48,006 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:48,007 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:48,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:49,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:50,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:52,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:53,470 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:08:54,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:59,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:00,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:00,659 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:04,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:08,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:08,475 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:08,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:08,476 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:08,698 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:10,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:10,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:12,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:15,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:15,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:15,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:20,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:23,471 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:09:26,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:26,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:26,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:26,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:26,347 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:26,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:28,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:30,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:30,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:30,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:31,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:36,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:41,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:45,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:46,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:46,700 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:46,700 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:46,701 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:46,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:46,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:48,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:50,714 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:51,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:53,472 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:09:57,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:00,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:00,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:00,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:00,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:00,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:00,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:01,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:02,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:02,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:04,719 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:06,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:08,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:13,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:15,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:18,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:19,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:19,268 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:19,268 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:19,269 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:19,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:20,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:23,473 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:10:24,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:24,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:29,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:30,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:30,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:34,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:39,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:41,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:41,161 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:41,162 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:41,162 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:41,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:42,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:45,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:45,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:45,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:46,736 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:50,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:53,474 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:10:56,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:00,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:01,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:02,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:02,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:02,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:02,519 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:02,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:04,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:06,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:07,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:12,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:15,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:17,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:19,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:19,261 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:19,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:19,262 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:19,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:20,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:23,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:23,475 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:11:24,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:28,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:30,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:33,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:38,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:39,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:39,141 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:39,142 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:39,142 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:39,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:40,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:44,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:44,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:45,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:49,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:53,476 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:11:55,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:57,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:57,867 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:57,867 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:57,867 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:58,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:58,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:00,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:02,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:06,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:11,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:15,184 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:15,185 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:15,186 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:15,186 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:15,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:15,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:15,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:16,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:16,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:20,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:22,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:23,477 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:12:27,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:30,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:32,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:32,167 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:32,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:32,168 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:32,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:33,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:34,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:36,779 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:38,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:43,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:45,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:45,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:48,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:48,522 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:48,522 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:48,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:48,523 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:48,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:50,784 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:52,785 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:53,478 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:12:54,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:59,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:00,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:03,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:03,256 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:03,256 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:03,257 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:03,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:04,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:05,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:08,791 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:11,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:15,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:15,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:15,665 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:16,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:21,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:23,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:23,206 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:23,206 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:23,207 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:23,479 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:13:23,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:24,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:27,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:28,798 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:30,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:30,665 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:32,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:37,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:42,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:42,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:42,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:42,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:42,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:43,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:44,804 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:45,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:45,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:46,805 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:48,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:53,480 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:13:54,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:59,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:00,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:00,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:04,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:06,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:06,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:06,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:06,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:06,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:06,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:10,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:10,813 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:15,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:15,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:15,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:20,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:20,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:20,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:20,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:20,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:20,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:22,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:23,481 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:14:24,819 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:25,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:30,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:30,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:31,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:36,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:39,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:39,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:39,347 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:39,347 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:39,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:40,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:42,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:44,830 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:45,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:45,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:47,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:52,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:53,482 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:14:55,199 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:55,200 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:55,201 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:55,201 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:55,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:56,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:58,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:58,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:00,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:00,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:03,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:08,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:11,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:11,516 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:11,516 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:11,517 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:11,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:12,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:14,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:15,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:15,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:15,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:16,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:19,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:23,483 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:15:25,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:26,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:26,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:26,344 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:26,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:26,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:26,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:30,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:30,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:30,710 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:30,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:35,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:40,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:45,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:45,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:46,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:51,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:52,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:52,330 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:52,330 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:52,331 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:52,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:52,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:53,483 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:15:56,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:57,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:00,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:00,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:00,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:02,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:07,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:10,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:10,182 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:10,182 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:10,183 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:10,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:10,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:13,197 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:14,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:15,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:15,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:15,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:18,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:23,484 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:16:24,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:29,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:30,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:30,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:30,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:32,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:32,689 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:32,690 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:32,690 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:32,873 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:34,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:34,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:36,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:39,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:44,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:45,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:45,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:45,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:50,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:50,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:50,392 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:50,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:50,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:50,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:51,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:53,485 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:16:55,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:55,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:00,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:00,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:00,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:00,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:04,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:04,150 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:04,150 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:04,151 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:04,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:05,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:06,184 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:09,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:11,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:15,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:15,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:15,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:16,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:21,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:22,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:22,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:22,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:22,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:22,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:23,486 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:17:25,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:27,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:27,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:30,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:30,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:30,676 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:32,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:37,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:40,173 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:40,175 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:40,175 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:40,176 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:40,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:41,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:43,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:45,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:45,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:45,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:45,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:48,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:53,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:54,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:56,067 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:17:56,068 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:56,068 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:56,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:56,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:57,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:59,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:59,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:00,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:00,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:00,681 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:04,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:09,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:13,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:13,746 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:13,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:13,747 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:13,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:15,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:15,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:16,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:16,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:16,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:19,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:20,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:23,488 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:18:26,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:31,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:31,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:31,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:31,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:36,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:36,058 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:36,058 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:36,060 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:36,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:37,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:37,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:41,926 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:42,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:46,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:46,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:46,386 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:48,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:53,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:53,489 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:18:53,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:53,666 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:53,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:53,667 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:53,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:55,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:58,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:59,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:01,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:01,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:04,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:09,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:10,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:10,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:10,154 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:10,156 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:10,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:11,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:15,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:15,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:16,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:16,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:21,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:23,490 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:19:23,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:23,792 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:23,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:23,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:23,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:25,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:26,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:29,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:31,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:31,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:31,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:32,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:41,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:41,198 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:41,198 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:41,200 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:41,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:43,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:43,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:45,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:46,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:46,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:48,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:53,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:53,490 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:19:58,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:59,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:59,599 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:59,599 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:59,599 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:59,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:01,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:01,959 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:03,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:03,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:05,961 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:08,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:13,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:16,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:18,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:18,096 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:18,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:18,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:18,965 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:19,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:19,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:23,491 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:20:23,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:24,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:29,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:31,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:31,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:31,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:32,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:32,902 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:32,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:32,902 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:32,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:34,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:35,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:37,973 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:40,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:45,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:46,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:49,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:49,397 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:49,397 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:49,399 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:49,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:50,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:51,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:53,492 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:20:53,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:55,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:00,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:01,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:06,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:11,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:13,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:21:13,832 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:21:13,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:21:13,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:21:13,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:21:15,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:16,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:16,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:17,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:19,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:22,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:23,493 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:21:27,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:31,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:31,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:31,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:32,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:33,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:21:33,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:21:33,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:21:33,782 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:21:33,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:21:35,996 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:37,996 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:38,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:39,997 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:43,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:46,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:46,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:46,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:48,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:53,494 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:21:54,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:59,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:01,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:01,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:01,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:01,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:01,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:01,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:01,766 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:02,005 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:04,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:04,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:06,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:08,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:10,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:15,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:16,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:16,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:16,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:20,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:23,495 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:22:23,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:23,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:23,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:23,829 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:24,013 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:25,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:26,014 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:28,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:30,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:31,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:31,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:31,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:36,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:40,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:40,400 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:40,401 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:40,401 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:41,020 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:42,020 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:42,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:46,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:46,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:46,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:46,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:47,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:52,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:53,502 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:22:58,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:01,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:01,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:01,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:03,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:05,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:05,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:05,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:05,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:06,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:08,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:08,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:10,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:12,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:14,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:16,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:16,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:19,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:21,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:21,266 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:21,266 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:21,267 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:22,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:23,498 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:23:24,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:24,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:26,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:29,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:31,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:31,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:31,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:35,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:40,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:44,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:44,775 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:44,775 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:44,776 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:45,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:46,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:46,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:46,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:50,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:51,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:53,498 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:23:56,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:01,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:01,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:01,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:01,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:03,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:03,228 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:03,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:03,230 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:04,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:06,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:07,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:08,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:12,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:16,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:16,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:16,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:17,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:22,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:23,500 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:24:25,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:25,305 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:25,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:25,306 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:26,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:28,060 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:28,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:30,061 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:31,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:33,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:38,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:42,024 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:42,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:42,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:42,027 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:42,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:44,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:44,066 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:46,067 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:46,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:49,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:53,500 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:24:55,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:00,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:01,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:01,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:01,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:01,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:01,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:02,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:04,074 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:06,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:06,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:11,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:16,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:16,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:16,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:16,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:21,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:23,502 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:25:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:25,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:25,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:25,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:26,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:27,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:28,082 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:30,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:31,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:32,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:37,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:42,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:46,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:46,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:46,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:48,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:48,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:48,793 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:48,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:48,794 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:49,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:50,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:53,503 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:25:54,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:54,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:59,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:01,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:01,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:04,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:09,154 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:09,155 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:09,155 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:09,155 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:10,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:10,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:10,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:14,099 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:15,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:16,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:16,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:16,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:20,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:23,504 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:26:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:25,237 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:25,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:25,238 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:26,104 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:26,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:26,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:30,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:31,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:31,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:36,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:41,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:46,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:46,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:46,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:47,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:51,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:51,320 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:51,321 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:51,321 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:52,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:52,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:53,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:53,505 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:26:56,116 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:58,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:01,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:01,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:01,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:03,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:07,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:07,608 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:07,609 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:07,609 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:08,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:08,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:10,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:12,123 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:13,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:16,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:16,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:16,508 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:19,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:23,505 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:27:25,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:29,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:29,472 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:29,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:29,473 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:30,130 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:30,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:32,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:32,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:32,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:32,131 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:34,132 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:35,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:40,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:44,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:44,847 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:44,847 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:44,848 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:45,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:45,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:46,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:47,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:47,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:47,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:50,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:51,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:53,507 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:27:56,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:01,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:02,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:02,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:02,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:02,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:02,831 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:02,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:02,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:03,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:04,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:07,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:08,144 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:12,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:15,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:15,883 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:15,883 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:15,883 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:16,148 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:17,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:17,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:18,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:20,150 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:23,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:23,508 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:28:28,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:32,587 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:33,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:37,470 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:37,470 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:37,471 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:38,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:38,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:39,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:42,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:44,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:47,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:50,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:53,509 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:28:56,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:01,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:02,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:02,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:02,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:06,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:06,108 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:06,108 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:06,109 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:06,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:07,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:08,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:10,169 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:12,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:17,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:17,587 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:17,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:22,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:23,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:29:24,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:24,294 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:24,295 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:24,295 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:25,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:26,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:28,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:28,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:32,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:32,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:33,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:38,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:38,252 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:38,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:38,252 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:39,179 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:39,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:40,180 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:42,180 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:44,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:47,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:47,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:49,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:53,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:29:55,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:55,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:55,932 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:55,932 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:55,933 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:56,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:58,186 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:00,187 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:01,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:02,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:02,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:02,589 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:06,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:11,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:11,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:11,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:11,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:12,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:12,192 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:12,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:16,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:17,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:17,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:17,622 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:17,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:22,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:23,512 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:30:26,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:26,878 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:26,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:26,878 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:27,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:27,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:28,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:32,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:32,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:32,593 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:32,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:33,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:38,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:43,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:45,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:45,579 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:45,579 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:45,579 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:46,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:46,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:48,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:48,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:48,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:48,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:50,207 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:53,513 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:30:54,514 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:59,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:03,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:03,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:03,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:04,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:04,260 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:04,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:05,253 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:05,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:06,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:06,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:08,214 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:10,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:15,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:18,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:18,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:18,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:19,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:19,033 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:19,033 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:19,033 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:19,218 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:20,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:21,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:23,514 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:31:24,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:26,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:31,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:33,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:33,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:33,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:36,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:36,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:36,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:36,271 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:37,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:37,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:38,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:40,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:42,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:47,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:48,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:48,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:48,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:53,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:53,514 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:31:53,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:53,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:53,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:53,532 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:54,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:54,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:58,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:58,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:03,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:03,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:03,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:04,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:09,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:10,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:10,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:10,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:10,689 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:11,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:12,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:14,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:14,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:16,240 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:18,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:18,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:18,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:20,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:23,515 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:32:25,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:28,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:28,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:28,654 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:28,654 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:29,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:30,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:30,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:32,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:33,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:33,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:33,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:36,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:41,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:46,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:46,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:46,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:46,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:46,831 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:47,254 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:48,254 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:48,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:48,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:48,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:50,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:51,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:52,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:54,707 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:32:57,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:02,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:03,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:03,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:03,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:08,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:10,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:10,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:10,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:10,499 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:11,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:12,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:13,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:14,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:18,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:18,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:18,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:19,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:23,517 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:33:25,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:30,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:30,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:30,927 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:30,928 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:30,929 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:31,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:32,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:33,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:33,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:33,657 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:36,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:36,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:41,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:46,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:47,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:47,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:47,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:47,519 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:48,277 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:48,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:49,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:51,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:52,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:53,518 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:33:57,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:02,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:04,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:04,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:04,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:06,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:06,499 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:06,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:06,501 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:07,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:07,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:08,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:10,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:12,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:17,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:19,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:19,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:19,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:23,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:23,519 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:34:25,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:25,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:25,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:25,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:25,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:27,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:29,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:31,298 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:34,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:34,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:34,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:34,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:39,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:44,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:48,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:48,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:48,366 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:48,368 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:49,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:49,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:49,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:49,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:51,306 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:53,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:53,520 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:34:54,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:59,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:04,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:04,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:05,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:05,240 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:05,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:05,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:05,241 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:05,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:07,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:10,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:11,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:15,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:19,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:19,226 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:19,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:21,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:23,521 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:35:25,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:25,140 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:25,140 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:25,141 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:25,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:27,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:27,320 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:31,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:32,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:34,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:34,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:34,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:38,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:40,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:40,691 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:40,691 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:40,691 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:41,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:43,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:43,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:45,327 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:49,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:49,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:49,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:49,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:53,522 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:35:54,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:57,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:57,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:57,601 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:57,601 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:58,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:59,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:59,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:03,334 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:04,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:04,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:04,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:05,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:10,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:15,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:19,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:19,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:19,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:20,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:20,806 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:20,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:20,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:20,807 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:21,341 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:23,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:23,523 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:36:25,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:26,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:31,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:34,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:34,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:34,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:36,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:39,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:39,040 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:39,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:39,040 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:39,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:41,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:42,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:43,350 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:47,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:49,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:49,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:49,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:52,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:53,524 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:36:54,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:54,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:54,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:54,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:55,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:57,355 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:57,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:59,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:02,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:04,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:04,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:04,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:08,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:13,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:14,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:14,224 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:14,224 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:14,225 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:14,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:15,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:18,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:19,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:19,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:19,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:19,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:23,525 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:37:24,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:29,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:31,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:31,591 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:31,591 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:31,592 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:32,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:33,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:34,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:34,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:34,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:34,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:37,372 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:39,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:44,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:49,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:50,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:53,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:53,493 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:53,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:53,494 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:53,526 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:37:54,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:55,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:55,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:59,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:00,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:04,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:06,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:11,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:14,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:14,188 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:14,188 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:14,188 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:38:14,386 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:38:15,386 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:19,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:19,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:19,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:19,388 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:22,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:23,527 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:38:27,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:32,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:32,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:32,989 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:32,990 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:32,990 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:38:33,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:38:34,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:34,722 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:34,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:35,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:37,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:38,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:39,396 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:43,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:48,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:49,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:49,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:49,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:53,528 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:38:53,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:58,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:59,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:59,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:59,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:59,822 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:00,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:01,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:03,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:04,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:04,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:04,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:05,407 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:09,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:14,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:19,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:19,258 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:19,258 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:19,259 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:19,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:19,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:19,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:19,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:20,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:21,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:23,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:23,529 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:39:26,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:31,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:34,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:36,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:41,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:41,564 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:41,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:41,564 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:42,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:42,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:43,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:47,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:47,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:49,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:49,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:52,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:53,529 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:39:58,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:59,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:59,055 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:59,055 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:59,056 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:59,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:01,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:03,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:04,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:04,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:04,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:09,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:14,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:19,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:19,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:19,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:20,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:23,531 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:40:25,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:27,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:40:27,345 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:40:27,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:40:27,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:40:27,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:29,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:31,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:31,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:34,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:34,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:34,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:36,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:45,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:40:45,361 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:40:45,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:40:45,361 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:40:45,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:47,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:47,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:49,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:49,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:49,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:49,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:52,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:53,532 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:40:58,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:02,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:02,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:02,070 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:02,070 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:02,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:03,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:04,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:04,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:04,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:07,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:09,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:14,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:17,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:17,239 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:17,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:17,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:17,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:19,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:19,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:19,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:19,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:19,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:21,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:23,533 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:41:24,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:29,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:34,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:34,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:34,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:39,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:43,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:43,371 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:43,372 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:43,372 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:43,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:45,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:45,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:47,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:49,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:49,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:50,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:53,534 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:41:56,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:01,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:01,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:01,788 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:01,789 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:01,789 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:02,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:03,478 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:04,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:04,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:04,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:06,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:07,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:11,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:16,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:18,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:18,888 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:18,888 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:18,888 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:19,484 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:20,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:20,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:20,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:21,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:22,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:23,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:23,534 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:42:27,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:32,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:35,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:35,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:35,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:37,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:39,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:39,341 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:39,341 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:39,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:39,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:41,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:43,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:43,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:48,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:50,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:50,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:50,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:53,535 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:42:54,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:56,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:56,373 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:56,373 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:56,373 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:56,498 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:57,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:00,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:01,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:05,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:05,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:05,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:05,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:10,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:15,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:16,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:16,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:16,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:16,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:17,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:20,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:20,685 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:20,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:20,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:21,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:23,536 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:43:26,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:31,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:31,190 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:31,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:31,211 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:31,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:32,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:33,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:35,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:35,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:35,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:35,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:37,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:42,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:47,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:50,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:50,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:50,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:51,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:51,408 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:51,408 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:51,409 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:51,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:53,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:53,520 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:53,537 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:43:55,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:58,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:03,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:05,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:05,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:05,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:08,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:11,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:11,855 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:11,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:11,856 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:12,527 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:13,528 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:13,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:15,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:17,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:18,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:20,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:20,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:20,675 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:23,538 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:44:24,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:26,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:26,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:26,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:26,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:27,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:29,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:29,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:31,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:35,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:35,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:35,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:35,716 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:40,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:44,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:44,506 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:44,506 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:44,507 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:44,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:45,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:46,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:49,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:50,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:50,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:50,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:51,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:53,539 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:44:57,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:01,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:01,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:01,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:01,718 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:02,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:02,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:03,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:05,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:05,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:05,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:05,689 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:08,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:13,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:18,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:20,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:20,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:20,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:21,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:21,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:21,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:21,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:21,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:23,540 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:45:23,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:24,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:25,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:29,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:34,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:35,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:35,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:35,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:37,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:37,897 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:37,898 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:37,898 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:38,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:39,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:39,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:41,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:43,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:45,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:50,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:50,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:50,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:50,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:52,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:52,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:52,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:52,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:53,541 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:45:53,566 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:53,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:55,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:57,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:00,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:05,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:05,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:05,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:06,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:11,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:16,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:18,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:46:18,890 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:46:18,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:46:18,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:46:19,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:46:19,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:21,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:21,482 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:21,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:21,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:23,542 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:46:23,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:27,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:32,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:36,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:36,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:36,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:37,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:42,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:45,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:46:45,939 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:46:45,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:46:45,940 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:46:46,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:46:47,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:47,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:49,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:51,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:51,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:51,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:53,543 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:46:53,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:58,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:02,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:02,843 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:02,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:02,843 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:03,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:03,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:03,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:06,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:06,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:06,500 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:07,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:09,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:14,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:19,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:21,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:21,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:21,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:23,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:23,360 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:23,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:23,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:23,544 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:47:23,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:25,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:25,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:27,602 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:30,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:36,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:36,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:36,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:36,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:41,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:43,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:43,806 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:43,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:43,807 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:44,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:45,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:46,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:47,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:51,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:51,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:51,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:52,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:53,545 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:47:58,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:01,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:01,122 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:01,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:01,123 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:01,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:01,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:05,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:06,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:06,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:06,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:09,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:14,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:16,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:16,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:16,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:16,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:17,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:19,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:21,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:21,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:21,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:21,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:23,546 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:48:25,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:30,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:33,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:33,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:33,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:33,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:34,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:35,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:35,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:36,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:36,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:37,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:40,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:45,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:50,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:51,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:51,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:51,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:53,547 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:48:53,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:53,580 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:53,580 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:53,581 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:53,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:55,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:56,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:57,636 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:01,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:06,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:06,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:06,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:07,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:12,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:14,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:14,706 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:14,706 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:14,706 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:17,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:19,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:21,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:21,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:21,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:23,548 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:49:23,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:28,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:33,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:36,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:36,278 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:36,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:36,279 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:36,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:36,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:36,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:36,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:37,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:38,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:39,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:41,653 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:44,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:49,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:51,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:51,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:51,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:53,549 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:49:54,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:55,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:55,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:55,440 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:55,440 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:55,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:57,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:59,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:59,661 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:04,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:06,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:06,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:06,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:10,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:15,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:15,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:15,745 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:15,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:15,746 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:16,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:17,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:19,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:20,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:21,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:21,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:21,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:23,550 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:50:26,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:31,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:36,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:36,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:38,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:38,106 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:38,106 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:38,107 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:38,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:39,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:41,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:42,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:47,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:51,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:51,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:51,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:52,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:53,551 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:50:57,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:57,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:57,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:57,301 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:57,681 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:57,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:58,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:01,683 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:03,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:06,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:06,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:06,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:08,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:11,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:11,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:11,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:11,165 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:11,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:11,687 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:14,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:15,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:19,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:21,550 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:23,552 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:51:24,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:29,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:32,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:32,677 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:32,678 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:32,678 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:32,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:33,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:34,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:36,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:36,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:36,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:37,696 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:40,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:45,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:47,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:47,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:47,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:47,499 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:47,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:47,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:50,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:51,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:51,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:51,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:51,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:53,553 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:51:56,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:01,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:03,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:03,137 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:03,137 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:03,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:03,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:03,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:06,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:06,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:06,549 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:06,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:07,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:11,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:16,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:17,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:17,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:17,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:17,313 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:17,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:17,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:21,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:21,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:21,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:22,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:22,714 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:23,554 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:52:28,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:33,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:33,452 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:33,453 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:33,453 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:33,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:34,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:34,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:36,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:36,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:36,549 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:38,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:39,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:44,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:49,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:49,311 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:49,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:49,312 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:49,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:50,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:50,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:51,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:51,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:51,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:53,555 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:52:54,726 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:55,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:00,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:05,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:06,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:06,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:06,915 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:08,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:08,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:08,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:08,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:09,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:10,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:10,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:14,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:16,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:21,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:21,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:21,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:21,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:23,556 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:53:26,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:26,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:26,843 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:26,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:26,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:27,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:28,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:31,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:32,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:36,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:36,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:36,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:37,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:40,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:40,035 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:40,036 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:40,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:40,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:42,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:42,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:44,744 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:47,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:51,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:51,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:51,917 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:52,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:53,557 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:53:58,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:00,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:00,289 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:00,289 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:00,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:00,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:02,751 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:03,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:04,752 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:06,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:06,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:06,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:09,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:14,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:19,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:20,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:20,638 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:20,639 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:20,639 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:20,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:22,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:22,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:22,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:22,759 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:23,558 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:54:24,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:24,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:26,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:34,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:34,614 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:34,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:34,615 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:34,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:35,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:36,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:37,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:37,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:37,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:38,765 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:40,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:45,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:50,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:51,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:51,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:51,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:51,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:51,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:52,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:52,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:52,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:53,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:53,559 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:54:56,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:56,772 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:06,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:07,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:07,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:07,365 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:07,365 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:07,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:07,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:07,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:08,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:08,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:11,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:12,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:16,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:22,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:22,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:23,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:23,559 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:55:26,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:27,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:27,391 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:27,391 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:27,391 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:27,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:28,784 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:31,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:32,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:36,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:37,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:37,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:38,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:42,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:43,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:43,374 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:43,375 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:43,375 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:43,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:44,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:47,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:48,792 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:52,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:52,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:52,998 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:53,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:53,561 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:55:57,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:02,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:04,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:04,809 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:04,809 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:04,809 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:04,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:06,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:07,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:07,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:08,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:08,001 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:08,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:10,819 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:13,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:18,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:22,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:22,028 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:22,028 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:22,029 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:22,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:23,562 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:56:23,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:23,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:23,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:23,771 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:24,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:26,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:29,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:34,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:38,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:38,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:38,770 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:39,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:39,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:39,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:39,180 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:39,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:40,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:44,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:45,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:50,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:53,562 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:56:53,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:53,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:53,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:55,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:58,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:58,497 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:58,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:58,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:58,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:00,841 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:01,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:02,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:06,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:08,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:08,776 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:08,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:12,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:17,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:22,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:22,136 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:22,136 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:22,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:22,850 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:23,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:23,563 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:57:23,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:23,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:23,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:24,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:26,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:28,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:33,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:38,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:38,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:38,789 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:38,789 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:38,790 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:38,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:38,877 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:38,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:38,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:40,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:44,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:44,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:49,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:53,564 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:57:53,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:53,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:54,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:54,360 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:54,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:54,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:54,361 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:54,864 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:58,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:59,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:04,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:08,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:08,883 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:08,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:09,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:09,574 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:09,575 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:09,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:09,575 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:09,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:10,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:14,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:14,872 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:19,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:23,566 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:58:23,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:23,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:23,920 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:25,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:29,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:29,156 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:29,156 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:29,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:29,878 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:30,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:30,878 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:34,880 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:35,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:38,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:38,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:38,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:41,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:46,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:50,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:50,442 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:50,442 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:50,442 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:50,886 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:51,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:52,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:53,566 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:58:53,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:53,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:53,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:54,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:57,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:02,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:05,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:05,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:05,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:05,857 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:05,892 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:06,893 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:07,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:08,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:10,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:13,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:18,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:23,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:23,568 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:59:23,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:23,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:23,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:24,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:24,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:24,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:24,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:24,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:26,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:28,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:28,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:33,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:38,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:38,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:38,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:44,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:48,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:48,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:48,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:48,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:48,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:48,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:49,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:52,911 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:53,569 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:59:53,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:53,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:54,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:59,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:01,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:01,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:01,798 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:02,827 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:02,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:04,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:05,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:06,916 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:08,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:10,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:15,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:20,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:23,570 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:00:23,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:23,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:23,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:25,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:25,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:25,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:25,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:25,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:26,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:26,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:28,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:31,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:36,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:38,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:38,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:41,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:42,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:42,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:42,315 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:42,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:42,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:44,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:46,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:47,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:52,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:53,571 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:00:53,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:53,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:57,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:02,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:02,504 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:02,504 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:02,505 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:02,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:03,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:04,939 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:06,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:08,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:08,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:13,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:18,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:20,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:20,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:20,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:20,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:20,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:22,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:23,571 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:01:23,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:23,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:23,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:23,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:24,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:28,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:33,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:38,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:38,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:38,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:42,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:42,559 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:42,559 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:42,559 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:42,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:44,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:44,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:46,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:49,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:53,572 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:01:53,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:53,893 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:53,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:54,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:59,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:01,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:01,889 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:01,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:01,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:01,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:02,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:04,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:06,962 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:08,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:08,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:08,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:09,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:14,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:19,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:23,573 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:02:23,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:23,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:23,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:24,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:24,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:24,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:24,518 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:24,968 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:25,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:26,969 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:28,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:30,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:35,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:38,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:38,893 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:38,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:40,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:43,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:43,724 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:43,724 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:43,725 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:43,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:44,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:46,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:48,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:52,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:53,574 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:02:53,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:53,937 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:53,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:57,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:57,425 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:57,426 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:57,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:57,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:57,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:00,982 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:02,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:02,983 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:07,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:08,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:08,897 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:08,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:13,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:18,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:20,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:03:20,466 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:03:20,466 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:03:20,466 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:03:20,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:20,990 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:23,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:23,575 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:03:23,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:23,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:23,941 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:24,991 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:29,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:34,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:38,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:38,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:38,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:39,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:44,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:03:44,050 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:03:44,050 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:03:44,051 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:03:44,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:44,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:45,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:49,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:50,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:53,576 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:03:53,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:53,926 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:53,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:56,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:01,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:03,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:03,993 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:03,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:03,993 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:04,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:05,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:06,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:08,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:08,907 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:08,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:09,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:12,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:17,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:22,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:23,577 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:04:23,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:23,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:23,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:25,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:25,436 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:25,436 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:25,437 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:26,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:27,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:27,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:31,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:32,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:37,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:38,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:38,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:38,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:42,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:42,609 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:42,609 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:42,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:42,610 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:43,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:45,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:47,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:47,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:52,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:53,577 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:04:53,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:53,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:53,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:58,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:58,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:58,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:58,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:58,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:59,028 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:01,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:03,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:03,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:08,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:08,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:08,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:08,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:14,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:19,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:20,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:20,889 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:20,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:20,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:21,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:23,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:23,579 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:05:23,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:23,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:23,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:24,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:25,039 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:29,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:34,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:37,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:37,257 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:37,257 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:37,257 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:38,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:38,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:38,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:38,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:39,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:40,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:43,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:45,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:50,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:53,580 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:05:53,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:53,982 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:53,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:54,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:54,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:54,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:54,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:55,050 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:56,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:57,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:59,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:01,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:06,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:08,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:08,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:09,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:11,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:11,092 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:11,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:11,093 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:12,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:12,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:13,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:17,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:17,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:22,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:23,581 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:06:23,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:23,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:23,990 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:25,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:25,550 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:25,551 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:25,551 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:26,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:27,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:28,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:31,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:33,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:38,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:38,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:38,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:38,989 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:44,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:46,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:46,810 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:46,810 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:46,810 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:47,069 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:49,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:49,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:51,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:53,581 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:06:53,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:53,989 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:53,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:55,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:00,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:02,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:02,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:02,528 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:02,806 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:03,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:05,076 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:05,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:07,076 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:09,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:09,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:09,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:11,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:16,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:17,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:17,853 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:17,853 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:17,853 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:18,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:19,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:21,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:23,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:23,582 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:07:23,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:23,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:24,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:27,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:32,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:37,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:38,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:38,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:39,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:42,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:42,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:42,819 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:42,820 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:42,820 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:43,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:45,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:47,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:48,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:53,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:53,583 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:07:53,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:54,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:54,001 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:58,129 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:59,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:59,170 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:59,170 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:59,171 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:00,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:03,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:03,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:08,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:08,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:08,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:09,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:14,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:19,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:21,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:21,973 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:21,973 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:21,974 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:22,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:23,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:23,585 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:08:24,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:24,571 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:24,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:24,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:27,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:30,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:35,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:38,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:38,490 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:38,490 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:38,490 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:39,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:39,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:40,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:40,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:40,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:40,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:43,113 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:45,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:50,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:53,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:53,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:53,513 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:53,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:53,586 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:08:54,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:55,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:55,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:55,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:55,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:56,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:57,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:59,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:01,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:06,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:10,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:12,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:14,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:14,756 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:14,756 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:14,757 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:09:15,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:09:17,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:17,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:19,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:22,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:23,587 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:09:25,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:25,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:25,232 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:28,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:33,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:35,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:35,646 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:35,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:35,646 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:09:36,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:09:37,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:38,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:40,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:40,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:40,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:41,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:44,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:49,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:53,588 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:09:54,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:55,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:55,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:55,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:59,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:59,547 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:59,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:59,548 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:00,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:00,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:01,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:03,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:05,147 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:05,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:10,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:10,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:10,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:11,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:16,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:21,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:23,588 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:10:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:25,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:26,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:10:26,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:10:26,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:10:26,270 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:27,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:27,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:27,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:31,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:32,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:37,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:40,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:40,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:43,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:45,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:10:45,375 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:10:45,375 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:10:45,375 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:46,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:47,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:48,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:49,169 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:53,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:53,589 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:10:55,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:55,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:55,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:59,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:03,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:03,047 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:03,047 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:03,048 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:03,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:05,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:06,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:08,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:10,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:10,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:10,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:10,237 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:15,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:20,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:23,591 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:11:25,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:25,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:25,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:25,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:27,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:27,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:27,514 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:27,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:28,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:30,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:30,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:32,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:35,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:40,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:40,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:41,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:42,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:42,835 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:42,835 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:42,835 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:43,189 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:46,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:47,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:48,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:52,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:53,592 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:11:55,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:55,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:55,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:57,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:59,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:59,143 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:59,143 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:59,153 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:59,195 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:02,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:02,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:04,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:07,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:10,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:10,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:10,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:13,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:17,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:12:17,611 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:12:17,612 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:12:17,612 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:12:18,202 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:18,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:20,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:22,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:23,593 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:12:24,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:25,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:25,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:25,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:30,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:35,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:40,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:40,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:41,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:41,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:12:41,523 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:12:41,524 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:12:41,524 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:12:42,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:44,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:46,214 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:46,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:51,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:53,594 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:12:55,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:55,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:55,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:57,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:01,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:01,753 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:01,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:01,754 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:02,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:02,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:04,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:06,222 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:07,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:10,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:10,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:10,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:13,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:18,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:19,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:19,068 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:19,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:19,069 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:19,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:22,227 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:23,595 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:13:23,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:24,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:25,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:25,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:25,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:29,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:34,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:37,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:37,339 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:37,339 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:37,339 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:38,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:39,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:40,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:40,257 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:40,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:42,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:44,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:49,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:53,596 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:13:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:55,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:55,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:55,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:00,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:02,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:02,087 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:02,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:02,088 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:02,243 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:04,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:05,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:08,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:10,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:10,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:10,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:11,294 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:16,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:21,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:21,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:21,831 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:21,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:21,831 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:22,252 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:23,597 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:14:24,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:25,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:25,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:25,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:26,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:26,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:31,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:36,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:36,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:36,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:36,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:37,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:37,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:38,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:40,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:40,250 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:40,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:42,260 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:42,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:47,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:52,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:53,597 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:14:55,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:55,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:55,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:58,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:58,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:58,664 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:58,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:58,664 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:59,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:00,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:03,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:04,268 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:08,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:10,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:10,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:10,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:14,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:19,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:21,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:21,121 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:21,122 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:21,122 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:21,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:23,598 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:15:24,275 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:25,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:25,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:25,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:25,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:26,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:30,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:35,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:35,080 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:35,080 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:35,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:35,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:36,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:36,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:40,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:40,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:40,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:40,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:41,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:46,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:51,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:53,599 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:15:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:55,269 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:55,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:55,733 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:55,733 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:55,733 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:56,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:56,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:58,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:00,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:01,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:06,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:10,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:10,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:12,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:16,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:16:16,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:16:16,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:16:16,547 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:16:17,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:16:17,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:18,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:22,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:22,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:23,599 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:16:25,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:25,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:25,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:28,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:33,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:38,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:40,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:40,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:40,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:42,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:16:42,525 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:16:42,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:16:42,525 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:16:43,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:16:43,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:44,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:48,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:48,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:53,600 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:16:54,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:55,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:55,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:55,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:00,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:00,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:00,468 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:00,468 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:00,469 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:01,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:02,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:05,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:06,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:10,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:10,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:10,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:11,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:16,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:18,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:18,299 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:18,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:18,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:18,318 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:20,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:21,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:23,601 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:17:24,320 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:25,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:25,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:25,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:26,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:31,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:35,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:35,981 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:35,981 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:35,982 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:36,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:37,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:38,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:40,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:40,272 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:40,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:40,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:42,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:47,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:52,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:53,480 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:53,481 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:53,481 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:53,482 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:53,602 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:17:54,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:56,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:56,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:56,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:56,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:57,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:58,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:02,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:07,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:11,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:11,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:11,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:13,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:16,129 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:16,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:16,131 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:16,131 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:16,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:18,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:18,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:20,340 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:23,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:23,608 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:18:26,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:26,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:26,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:29,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:34,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:34,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:34,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:34,086 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:34,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:35,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:36,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:38,346 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:40,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:41,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:46,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:51,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:51,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:51,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:51,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:51,668 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:52,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:53,605 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:18:54,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:56,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:56,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:56,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:01,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:06,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:08,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:08,171 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:08,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:08,172 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:08,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:10,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:11,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:11,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:11,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:11,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:12,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:16,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:21,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:23,606 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:19:26,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:26,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:26,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:27,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:29,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:29,248 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:29,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:29,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:29,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:30,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:32,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:34,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:37,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:41,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:41,092 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:43,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:45,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:45,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:45,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:45,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:45,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:46,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:48,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:50,375 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:53,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:53,606 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:19:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:56,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:56,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:59,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:04,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:07,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:07,482 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:07,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:07,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:08,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:08,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:09,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:11,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:11,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:11,094 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:12,383 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:15,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:20,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:23,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:23,056 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:23,056 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:23,056 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:23,387 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:23,608 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:20:24,387 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:25,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:26,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:26,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:26,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:28,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:31,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:36,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:41,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:41,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:41,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:42,147 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:43,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:43,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:43,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:43,486 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:44,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:44,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:47,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:48,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:52,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:53,609 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:20:56,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:56,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:56,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:58,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:03,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:03,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:03,982 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:03,983 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:03,983 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:04,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:06,404 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:08,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:08,404 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:11,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:11,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:11,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:13,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:18,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:23,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:23,610 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:21:25,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:25,057 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:25,057 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:25,058 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:25,411 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:26,411 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:26,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:26,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:26,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:29,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:30,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:34,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:39,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:41,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:44,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:46,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:46,317 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:46,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:46,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:46,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:48,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:50,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:50,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:53,610 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:21:55,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:56,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:56,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:56,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:00,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:05,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:05,697 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:05,697 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:05,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:06,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:06,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:08,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:10,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:11,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:11,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:11,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:11,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:16,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:21,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:23,612 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:22:24,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:24,065 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:24,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:24,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:24,434 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:26,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:26,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:26,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:26,795 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:27,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:28,436 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:32,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:37,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:41,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:41,785 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:41,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:42,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:43,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:43,633 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:43,634 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:43,634 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:44,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:44,443 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:48,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:48,444 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:53,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:53,612 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:22:56,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:56,785 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:56,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:58,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:03,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:03,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:03,923 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:03,923 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:03,924 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:04,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:06,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:08,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:09,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:11,921 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:14,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:19,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:23,613 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:23:25,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:26,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:26,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:26,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:27,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:27,138 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:27,138 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:27,138 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:27,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:28,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:31,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:32,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:36,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:41,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:41,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:41,326 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:41,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:41,326 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:41,465 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:41,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:41,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:41,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:42,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:46,467 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:46,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:51,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:53,614 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:23:56,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:56,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:56,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:56,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:58,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:58,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:58,600 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:58,600 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:59,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:00,473 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:02,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:02,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:04,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:07,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:11,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:11,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:11,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:12,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:17,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:22,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:22,547 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:22,547 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:22,548 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:23,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:23,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:23,615 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:24:24,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:26,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:26,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:26,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:28,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:33,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:38,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:39,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:39,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:39,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:39,242 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:39,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:40,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:41,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:41,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:41,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:44,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:44,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:49,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:53,616 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:24:53,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:53,646 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:53,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:53,647 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:54,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:54,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:54,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:56,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:56,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:56,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:58,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:59,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:04,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:09,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:11,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:11,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:11,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:11,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:11,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:11,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:11,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:11,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:12,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:15,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:16,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:20,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:23,617 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:25:25,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:26,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:26,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:26,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:31,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:35,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:35,845 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:35,845 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:35,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:36,509 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:36,509 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:36,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:40,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:41,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:41,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:47,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:52,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:52,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:52,592 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:52,592 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:52,593 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:53,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:53,617 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:25:54,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:56,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:56,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:56,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:58,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:03,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:08,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:11,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:11,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:12,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:12,712 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:12,712 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:12,713 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:13,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:13,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:14,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:16,525 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:18,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:23,618 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:26:24,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:26,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:26,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:26,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:29,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:33,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:33,681 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:33,681 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:33,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:34,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:34,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:35,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:38,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:40,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:41,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:41,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:49,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:49,459 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:49,459 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:49,460 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:49,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:50,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:51,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:53,619 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:26:54,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:56,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:56,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:56,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:01,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:03,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:03,760 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:03,760 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:03,761 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:04,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:04,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:06,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:08,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:11,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:11,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:12,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:17,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:22,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:23,620 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:27:26,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:26,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:27,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:28,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:28,429 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:28,429 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:28,429 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:28,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:30,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:32,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:32,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:37,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:41,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:41,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:42,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:45,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:45,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:45,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:45,009 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:45,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:46,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:48,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:48,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:53,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:53,621 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:27:56,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:56,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:58,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:02,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:02,177 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:02,178 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:02,178 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:04,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:06,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:09,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:11,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:11,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:15,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:20,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:20,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:20,938 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:20,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:20,939 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:21,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:22,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:23,622 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:28:24,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:25,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:26,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:26,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:26,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:30,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:35,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:37,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:37,921 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:37,922 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:37,922 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:38,585 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:38,585 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:40,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:41,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:41,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:42,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:45,997 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:50,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:53,624 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:28:56,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:56,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:56,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:58,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:58,802 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:58,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:58,803 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:59,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:01,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:01,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:03,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:07,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:11,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:11,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:11,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:12,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:17,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:21,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:29:21,224 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:29:21,224 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:29:21,225 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:29:21,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:23,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:23,605 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:23,625 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:29:25,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:26,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:26,913 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:27,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:28,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:33,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:38,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:41,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:41,913 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:44,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:44,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:29:44,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:29:44,315 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:29:44,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:29:44,613 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:45,613 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:49,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:49,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:53,626 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:29:54,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:56,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:56,943 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:56,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:00,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:02,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:02,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:02,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:02,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:02,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:03,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:05,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:07,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:10,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:11,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:11,929 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:11,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:16,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:21,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:22,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:22,981 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:22,981 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:22,982 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:23,627 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:30:23,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:25,631 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:26,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:26,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:26,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:26,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:27,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:32,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:37,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:38,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:38,665 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:38,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:38,666 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:39,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:41,638 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:41,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:41,932 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:41,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:42,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:43,639 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:47,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:53,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:53,628 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:30:54,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:54,214 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:54,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:54,214 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:54,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:55,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:57,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:57,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:57,017 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:58,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:59,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:03,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:08,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:11,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:11,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:11,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:13,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:13,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:13,023 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:13,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:31:13,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:31:14,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:15,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:17,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:19,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:23,629 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:31:24,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:26,952 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:26,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:29,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:34,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:39,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:42,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:42,368 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:42,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:42,369 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:31:42,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:31:43,669 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:45,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:47,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:50,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:53,630 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:31:55,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:56,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:59,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:59,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:59,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:59,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:00,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:00,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:01,676 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:05,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:05,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:10,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:11,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:16,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:20,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:32:20,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:32:20,872 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:32:20,872 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:21,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:21,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:23,631 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:32:23,683 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:25,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:26,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:26,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:32,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:37,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:41,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:41,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:41,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:43,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:43,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:32:43,772 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:32:43,773 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:32:43,773 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:44,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:45,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:48,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:49,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:53,632 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:32:54,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:56,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:00,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:04,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:04,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:04,130 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:04,130 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:04,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:05,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:05,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:09,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:10,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:11,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:11,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:11,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:16,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:20,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:20,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:20,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:20,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:20,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:21,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:21,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:23,633 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:33:25,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:26,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:26,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:26,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:31,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:36,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:36,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:36,395 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:36,395 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:36,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:37,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:37,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:41,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:41,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:43,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:48,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:53,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:53,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:53,603 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:53,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:53,604 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:53,633 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:33:53,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:55,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:56,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:56,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:57,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:59,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:59,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:04,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:09,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:10,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:10,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:10,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:10,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:11,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:12,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:13,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:14,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:15,729 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:19,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:23,634 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:34:24,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:27,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:27,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:27,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:27,087 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:27,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:27,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:27,602 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:27,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:29,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:29,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:31,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:35,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:40,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:42,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:42,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:45,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:47,024 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:47,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:47,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:47,026 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:47,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:49,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:51,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:51,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:53,635 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:34:56,636 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:57,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:57,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:57,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:01,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:06,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:07,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:07,221 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:07,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:07,221 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:07,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:09,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:11,751 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:12,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:12,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:12,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:17,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:22,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:23,636 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:35:27,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:27,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:27,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:27,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:32,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:33,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:33,064 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:33,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:33,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:33,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:35,759 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:37,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:38,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:42,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:42,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:43,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:48,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:50,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:50,216 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:50,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:50,216 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:50,765 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:51,766 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:53,637 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:35:54,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:55,767 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:57,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:57,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:57,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:59,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:04,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:09,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:12,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:12,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:12,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:12,701 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:12,701 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:12,702 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:12,774 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:13,774 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:15,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:17,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:20,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:23,638 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:36:26,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:27,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:27,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:27,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:27,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:27,888 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:27,888 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:27,888 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:28,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:29,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:31,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:32,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:33,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:37,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:42,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:42,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:42,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:42,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:44,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:44,658 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:44,658 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:44,658 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:44,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:45,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:47,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:49,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:53,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:53,639 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:36:57,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:57,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:57,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:58,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:03,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:04,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:04,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:04,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:04,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:04,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:05,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:09,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:09,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:12,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:12,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:12,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:14,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:19,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:20,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:20,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:20,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:20,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:20,800 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:21,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:23,639 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:37:25,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:25,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:27,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:27,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:27,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:30,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:35,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:40,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:40,565 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:40,565 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:40,566 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:40,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:41,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:41,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:43,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:43,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:43,226 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:45,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:46,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:51,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:53,640 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:37:57,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:58,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:58,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:58,264 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:58,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:58,721 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:58,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:58,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:58,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:59,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:02,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:03,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:07,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:12,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:13,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:13,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:13,229 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:18,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:21,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:21,386 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:21,386 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:21,386 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:21,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:23,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:23,641 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:38:23,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:25,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:28,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:28,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:28,229 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:29,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:34,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:39,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:40,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:40,895 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:40,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:40,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:41,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:41,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:43,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:43,558 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:44,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:47,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:50,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:53,642 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:38:55,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:58,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:58,217 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:58,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:58,217 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:58,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:58,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:58,556 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:58,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:59,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:01,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:03,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:06,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:11,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:13,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:13,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:13,558 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:16,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:21,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:22,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:39:22,985 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:39:22,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:39:22,986 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:39:23,643 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:39:23,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:39:23,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:27,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:27,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:28,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:28,556 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:28,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:32,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:37,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:41,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:39:41,028 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:39:41,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:39:41,029 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:39:41,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:39:41,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:43,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:43,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:43,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:43,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:45,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:48,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:53,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:53,647 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:39:58,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:58,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:58,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:58,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:00,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:00,306 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:00,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:00,306 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:00,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:01,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:03,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:05,867 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:08,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:13,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:13,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:13,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:13,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:18,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:23,644 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:40:24,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:28,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:28,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:28,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:29,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:31,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:31,726 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:31,726 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:31,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:31,879 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:33,880 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:34,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:35,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:39,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:43,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:43,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:43,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:45,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:47,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:47,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:47,484 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:47,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:47,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:49,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:50,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:51,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:53,645 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:40:56,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:58,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:58,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:58,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:01,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:03,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:03,917 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:03,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:03,920 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:04,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:05,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:06,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:07,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:12,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:13,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:13,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:13,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:17,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:22,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:23,646 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:41:27,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:27,707 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:27,708 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:27,708 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:27,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:28,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:29,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:29,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:29,903 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:31,904 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:33,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:38,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:43,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:44,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:44,446 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:44,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:46,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:46,827 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:46,828 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:46,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:46,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:47,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:49,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:51,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:53,647 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:41:55,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:59,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:01,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:06,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:06,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:06,909 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:06,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:06,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:06,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:07,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:11,920 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:12,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:14,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:14,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:14,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:17,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:22,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:22,333 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:22,333 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:22,336 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:22,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:23,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:23,648 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:42:23,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:25,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:28,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:29,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:29,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:33,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:38,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:40,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:40,502 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:40,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:40,502 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:40,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:41,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:43,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:44,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:44,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:44,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:45,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:49,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:53,649 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:42:54,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:59,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:59,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:59,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:00,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:02,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:02,762 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:02,763 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:02,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:02,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:03,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:05,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:07,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:11,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:14,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:14,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:14,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:16,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:21,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:22,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:22,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:22,153 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:22,153 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:22,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:23,649 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:43:23,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:25,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:27,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:29,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:29,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:32,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:36,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:36,730 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:36,730 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:36,730 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:36,955 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:37,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:37,955 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:41,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:42,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:44,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:44,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:44,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:48,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:53,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:53,650 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:43:56,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:56,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:56,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:56,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:56,963 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:57,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:58,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:59,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:59,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:59,526 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:01,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:03,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:08,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:11,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:11,402 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:11,402 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:11,402 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:11,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:11,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:14,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:14,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:14,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:14,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:15,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:19,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:23,652 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:44:24,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:26,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:26,238 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:26,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:26,239 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:26,975 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:27,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:29,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:29,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:29,458 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:29,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:30,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:35,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:40,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:44,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:44,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:44,477 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:46,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:49,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:49,139 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:49,139 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:49,139 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:49,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:49,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:52,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:53,652 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:44:53,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:57,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:59,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:59,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:59,464 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:03,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:07,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:07,085 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:07,085 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:07,086 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:07,991 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:07,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:09,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:11,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:14,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:14,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:14,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:14,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:19,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:23,654 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:45:24,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:26,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:26,862 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:26,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:26,863 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:26,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:27,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:29,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:29,585 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:29,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:29,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:32,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:34,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:39,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:44,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:44,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:45,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:45,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:45,975 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:45,976 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:45,976 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:46,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:48,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:50,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:51,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:53,655 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:45:56,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:59,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:59,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:59,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:02,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:06,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:06,426 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:06,426 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:06,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:07,014 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:08,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:08,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:10,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:13,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:14,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:14,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:14,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:19,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:23,656 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:46:24,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:26,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:26,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:26,654 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:26,654 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:27,021 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:28,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:29,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:29,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:29,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:29,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:30,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:34,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:39,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:44,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:44,034 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:44,034 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:44,035 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:44,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:44,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:45,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:45,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:46,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:48,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:51,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:53,656 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:46:56,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:59,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:59,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:59,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:02,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:04,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:04,463 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:04,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:04,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:05,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:07,039 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:07,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:09,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:12,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:14,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:14,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:14,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:18,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:23,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:23,657 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:47:26,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:26,770 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:26,770 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:26,771 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:27,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:28,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:29,048 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:29,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:29,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:29,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:31,049 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:33,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:38,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:42,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:42,476 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:42,476 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:42,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:43,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:45,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:45,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:45,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:45,102 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:47,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:49,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:53,658 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:47:55,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:58,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:58,476 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:58,477 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:58,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:59,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:00,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:00,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:00,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:01,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:01,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:03,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:06,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:11,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:14,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:14,801 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:14,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:14,802 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:15,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:15,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:15,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:15,178 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:17,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:17,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:21,079 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:23,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:23,659 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:48:28,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:30,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:30,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:30,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:34,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:34,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:34,632 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:34,632 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:34,632 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:35,084 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:37,085 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:39,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:39,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:44,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:45,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:45,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:45,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:49,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:49,295 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:49,295 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:49,295 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:50,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:50,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:51,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:53,660 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:48:55,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:55,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:00,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:00,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:00,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:01,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:06,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:11,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:14,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:14,004 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:14,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:14,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:14,100 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:15,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:15,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:15,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:15,755 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:16,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:19,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:22,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:23,661 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:49:27,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:30,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:30,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:30,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:31,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:31,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:31,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:31,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:32,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:32,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:33,108 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:37,110 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:37,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:42,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:45,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:45,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:45,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:48,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:49,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:49,310 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:49,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:49,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:50,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:51,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:53,662 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:49:54,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:55,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:59,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:00,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:00,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:00,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:06,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:06,805 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:06,805 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:06,805 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:07,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:09,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:09,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:11,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:14,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:16,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:16,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:16,064 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:20,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:22,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:22,630 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:22,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:22,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:23,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:23,663 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:50:25,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:25,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:27,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:30,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:31,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:31,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:36,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:39,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:39,764 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:39,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:39,764 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:40,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:41,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:41,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:45,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:46,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:46,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:46,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:47,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:52,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:53,664 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:50:54,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:54,082 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:54,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:54,083 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:54,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:55,139 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:57,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:59,140 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:01,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:01,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:01,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:03,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:08,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:10,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:10,199 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:10,199 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:10,199 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:11,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:11,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:13,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:15,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:16,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:16,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:16,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:19,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:23,665 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:51:24,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:29,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:30,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:30,413 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:30,413 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:30,413 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:31,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:31,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:31,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:33,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:35,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:35,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:40,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:45,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:46,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:46,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:46,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:49,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:49,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:49,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:49,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:50,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:51,160 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:51,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:53,665 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:51:55,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:56,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:01,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:01,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:01,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:02,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:03,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:03,594 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:03,595 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:03,595 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:04,164 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:05,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:07,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:09,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:12,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:16,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:16,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:16,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:18,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:21,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:21,172 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:21,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:21,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:22,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:23,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:23,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:23,666 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:52:25,172 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:28,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:31,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:31,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:34,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:39,112 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:40,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:40,661 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:40,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:40,662 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:41,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:43,178 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:44,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:45,179 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:46,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:46,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:46,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:50,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:53,667 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:52:55,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:00,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:01,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:01,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:01,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:03,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:03,244 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:03,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:03,245 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:04,186 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:05,187 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:06,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:07,188 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:11,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:16,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:16,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:16,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:17,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:22,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:23,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:23,022 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:23,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:23,023 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:23,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:23,668 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:53:25,194 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:27,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:27,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:31,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:31,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:31,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:33,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:38,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:40,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:40,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:40,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:40,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:41,201 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:43,202 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:43,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:45,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:46,011 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:46,011 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:46,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:49,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:53,669 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:53:54,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:58,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:58,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:58,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:58,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:59,216 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:59,216 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:00,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:01,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:01,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:01,074 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:03,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:05,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:10,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:12,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:12,993 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:12,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:12,994 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:13,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:15,222 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:16,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:16,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:16,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:16,033 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:17,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:21,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:23,670 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:54:26,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:30,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:30,974 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:30,975 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:30,975 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:31,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:31,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:31,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:31,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:32,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:33,229 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:35,230 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:37,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:37,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:42,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:46,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:46,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:46,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:47,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:47,556 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:47,557 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:47,557 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:48,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:48,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:49,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:53,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:53,671 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:54:53,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:58,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:01,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:01,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:01,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:04,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:09,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:09,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:09,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:09,337 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:09,338 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:10,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:11,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:13,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:14,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:16,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:16,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:16,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:20,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:23,672 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:55:25,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:26,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:26,720 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:26,720 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:26,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:27,251 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:29,252 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:30,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:31,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:31,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:31,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:31,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:36,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:41,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:45,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:45,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:45,798 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:45,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:46,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:46,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:46,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:46,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:47,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:47,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:51,261 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:53,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:53,673 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:55:58,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:01,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:01,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:01,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:01,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:01,667 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:01,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:01,668 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:02,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:03,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:03,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:05,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:07,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:08,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:13,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:16,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:16,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:16,042 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:19,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:23,674 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:56:23,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:23,832 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:23,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:23,833 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:24,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:24,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:25,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:29,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:29,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:31,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:31,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:31,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:35,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:40,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:40,122 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:40,122 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:40,122 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:40,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:40,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:41,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:45,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:45,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:46,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:46,037 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:51,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:53,675 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:56:56,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:57,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:57,840 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:57,841 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:57,841 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:58,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:59,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:01,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:01,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:01,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:02,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:03,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:07,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:12,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:16,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:16,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:16,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:18,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:19,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:19,698 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:19,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:19,699 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:20,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:21,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:23,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:23,676 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:57:23,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:25,298 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:28,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:31,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:31,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:31,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:34,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:35,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:35,578 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:35,578 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:35,579 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:36,302 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:37,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:39,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:39,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:44,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:46,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:46,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:50,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:53,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:53,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:53,024 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:53,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:53,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:53,676 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:57:55,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:55,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:57,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:01,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:01,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:01,075 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:01,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:06,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:10,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:10,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:10,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:10,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:10,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:11,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:11,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:15,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:16,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:16,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:16,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:17,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:22,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:23,677 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:58:27,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:31,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:31,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:31,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:31,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:31,123 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:31,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:31,124 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:31,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:33,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:33,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:35,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:38,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:46,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:46,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:46,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:48,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:48,621 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:48,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:48,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:48,622 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:49,329 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:49,329 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:53,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:53,679 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:58:53,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:58,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:01,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:01,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:01,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:04,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:09,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:09,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:09,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:09,392 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:09,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:10,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:11,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:13,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:14,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:16,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:16,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:16,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:20,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:23,680 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:59:25,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:28,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:28,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:28,054 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:28,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:29,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:31,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:31,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:31,064 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:31,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:33,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:36,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:41,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:46,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:46,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:46,061 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:47,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:47,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:47,054 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:47,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:47,055 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:47,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:49,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:51,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:52,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:53,680 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:59:57,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:01,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:01,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:01,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:03,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:04,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:04,564 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:04,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:04,565 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:05,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:05,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:08,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:09,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:13,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:16,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:16,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:16,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:19,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:21,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:21,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:21,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:21,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:21,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:21,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:23,681 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:00:24,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:25,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:29,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:31,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:31,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:31,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:35,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:40,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:43,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:43,526 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:43,526 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:43,527 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:44,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:45,374 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:45,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:46,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:46,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:46,195 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:47,375 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:51,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:53,683 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:00:56,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:01,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:01,196 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:01,199 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:02,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:03,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:03,759 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:03,760 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:03,760 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:04,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:05,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:07,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:07,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:12,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:16,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:16,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:16,193 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:18,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:23,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:23,684 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:01:28,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:29,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:29,263 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:29,263 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:29,263 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:29,390 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:31,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:31,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:31,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:33,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:34,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:39,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:44,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:46,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:49,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:53,685 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:01:54,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:54,631 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:54,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:54,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:55,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:55,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:55,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:59,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:01,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:01,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:06,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:11,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:16,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:16,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:16,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:17,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:19,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:02:19,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:02:19,023 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:02:19,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:02:19,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:02:19,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:23,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:23,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:23,686 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:02:28,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:31,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:31,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:31,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:33,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:38,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:41,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:02:41,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:02:41,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:02:41,444 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:02:42,418 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:02:43,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:44,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:45,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:49,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:53,687 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:02:55,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:00,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:01,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:01,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:01,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:04,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:04,535 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:04,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:04,535 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:05,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:05,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:06,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:09,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:11,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:16,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:16,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:16,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:18,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:18,467 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:18,468 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:18,468 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:19,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:19,434 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:22,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:23,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:23,688 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:03:27,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:31,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:31,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:31,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:32,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:36,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:36,052 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:36,052 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:36,053 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:36,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:37,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:38,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:39,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:43,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:46,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:46,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:48,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:53,694 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:03:54,695 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:56,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:56,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:56,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:56,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:57,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:57,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:59,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:01,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:01,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:01,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:04,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:09,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:13,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:04:13,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:04:13,445 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:04:13,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:04:13,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:04:15,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:15,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:16,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:16,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:16,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:17,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:20,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:23,690 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:04:26,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:31,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:31,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:36,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:37,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:04:37,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:04:37,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:04:37,446 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:04:37,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:04:39,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:41,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:42,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:46,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:47,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:52,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:53,690 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:04:58,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:01,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:01,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:02,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:02,623 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:02,623 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:02,623 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:03,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:04,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:04,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:08,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:09,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:16,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:16,866 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:16,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:19,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:23,691 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:05:24,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:24,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:24,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:24,345 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:24,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:25,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:26,478 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:30,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:30,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:35,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:40,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:41,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:41,091 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:41,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:41,093 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:41,483 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:44,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:46,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:46,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:46,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:51,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:53,693 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:05:57,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:01,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:02,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:03,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:03,645 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:03,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:03,646 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:04,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:06,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:08,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:08,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:13,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:16,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:16,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:18,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:21,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:21,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:21,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:21,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:21,498 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:23,693 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:06:24,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:24,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:26,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:29,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:34,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:39,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:39,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:39,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:39,554 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:40,505 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:40,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:42,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:44,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:45,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:46,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:51,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:06:56,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:01,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:01,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:03,105 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:03,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:03,106 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:03,108 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:03,109 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:03,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:04,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:06,515 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:07,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:08,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:12,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:13,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:18,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:19,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:19,209 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:19,210 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:19,210 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:19,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:22,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:22,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:23,696 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:23,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:24,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:27,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:28,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:31,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:33,698 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:37,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:38,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:41,954 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:41,956 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:42,527 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:42,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:43,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:44,528 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:46,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:46,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:49,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:52,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:53,697 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:54,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:57,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:58,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:58,792 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:58,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:58,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:59,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:00,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:01,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:04,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:04,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:07,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:09,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:12,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:14,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:16,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:17,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:17,307 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:17,307 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:17,307 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:17,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:08:20,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:20,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:22,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:22,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:23,697 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:08:25,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:27,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:30,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:35,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:36,962 WARNING FileStreamThread:1091 [file_stream.py:request_with_retry():687] requests_with_retry encountered retryable exception: 500 Server Error: Internal Server Error for url: https://api.wandb.ai/files/ducminh-ml71/huggingface/rimyh9a1/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/ducminh-ml71/huggingface/rimyh9a1/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"train/loss": 3.3958, "train/grad_norm": 99.48656463623047, "train/learning_rate": 1.2869649049440873e-05, "train/epoch": 0.05, "train/global_step": 11180, "_timestamp": 1712819261.9524338, "_runtime": 10819.103787899017, "_step": 564}']}, 'wandb-history.jsonl': {'offset': 562, 'content': ['{"train/loss": 3.2126, "train/grad_norm": 58.60544967651367, "train/learning_rate": 1.2965870820261672e-05, "train/epoch": 0.05, "train/global_step": 11140, "_timestamp": 1712819221.2956657, "_runtime": 10778.447019815445, "_step": 562}', '{"train/loss": 4.4423, "train/grad_norm": 15.332870483398438, "train/learning_rate": 1.2917749013937503e-05, "train/epoch": 0.05, "train/global_step": 11160, "_timestamp": 1712819239.2079432, "_runtime": 10796.359297275543, "_step": 563}', '{"train/loss": 3.3958, "train/grad_norm": 99.48656463623047, "train/learning_rate": 1.2869649049440873e-05, "train/epoch": 0.05, "train/global_step": 11180, "_timestamp": 1712819261.9524338, "_runtime": 10819.103787899017, "_step": 564}']}, 'wandb-events.jsonl': {'offset': 358, 'content': ['{"system.network.sent": 676280115.93, "system.network.recv": 452830156.0, "system.proc.memory.availableMB": 28276.28, "system.memory": 11.95, "system.proc.memory.rssMB": 2828.42, "system.proc.memory.percent": 8.81, "system.cpu": 28.48, "system.cpu.0.cpu_percent": 25.15, "system.cpu.1.cpu_percent": 61.31, "system.cpu.2.cpu_percent": 15.83, "system.cpu.3.cpu_percent": 21.72, "system.proc.cpu.threads": 43, "system.disk.\\\\.usagePercent": 69.0, "system.disk.\\\\.usageGB": 5566.96, "system.disk.in": 229.0, "system.disk.out": 976402.3, "system.gpu.0.memoryAllocated": 84.13, "system.gpu.1.memoryAllocated": 28.55, "system.gpu.0.memoryAllocatedBytes": 13550682112.0, "system.gpu.1.memoryAllocatedBytes": 4597940224.0, "system.gpu.0.memory": 33.13, "system.gpu.1.memory": 1.93, "system.gpu.0.gpu": 59.8, "system.gpu.1.gpu": 21.33, "system.gpu.0.temp": 76.93, "system.gpu.1.temp": 55.6, "system.gpu.0.powerWatts": 62.3, "system.gpu.1.powerWatts": 33.69, "system.gpu.0.powerPercent": 89.55, "system.gpu.1.powerPercent": 47.98, "_wandb": true, "_timestamp": 1712819243.694884, "_runtime": 10800.846238}']}, 'output.log': {'offset': 3380, 'content': ['ERROR 2024-04-11T07:07:03.631961 /opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n', 'ERROR 2024-04-11T07:07:03.631961 warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.")\n', 'ERROR 2024-04-11T07:07:05.636647 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n', 'ERROR 2024-04-11T07:07:05.636647 warnings.warn(\n', 'ERROR 2024-04-11T07:07:05.636647 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n', "ERROR 2024-04-11T07:07:05.636647 warnings.warn('Was asked to gather along dimension 0, but all '\n", 'ERROR 2024-04-11T07:07:21.656253 /opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n', 'ERROR 2024-04-11T07:07:21.656253 warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.")\n', 'ERROR 2024-04-11T07:07:23.661960 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n', 'ERROR 2024-04-11T07:07:23.661960 warnings.warn(\n', 'ERROR 2024-04-11T07:07:23.661960 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n', "ERROR 2024-04-11T07:07:23.661960 warnings.warn('Was asked to gather along dimension 0, but all '\n"]}}, 'dropped': 0}} +2024-04-11 07:08:37,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:38,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:38,651 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:38,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:38,651 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:39,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:08:40,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:41,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:42,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:44,551 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:46,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:46,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:51,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:52,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:53,698 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:08:57,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:57,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:57,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:57,896 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:57,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:57,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:58,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:00,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:01,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:02,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:03,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:07,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:08,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:12,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:13,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:16,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:18,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:18,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:09:18,683 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:09:18,683 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:09:18,684 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:09:19,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:20,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:22,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:23,699 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:09:23,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:24,566 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:27,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:28,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:33,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:37,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:38,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:40,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:09:40,762 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:09:40,763 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:09:40,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:09:41,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:42,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:42,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:43,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:46,574 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:49,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:52,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:53,701 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:09:54,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:57,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:59,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:01,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:03,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:03,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:03,511 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:03,512 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:03,581 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:05,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:06,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:07,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:08,583 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:10,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:12,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:15,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:16,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:20,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:22,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:23,702 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:10:26,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:26,322 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:26,323 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:26,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:26,324 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:26,590 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:28,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:28,591 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:30,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:31,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:32,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:36,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:37,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:41,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:42,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:46,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:46,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:47,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:47,470 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:47,470 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:47,472 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:47,598 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:50,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:51,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:52,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:52,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:53,703 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:10:56,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:57,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:01,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:01,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:06,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:07,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:07,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:07,397 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:07,398 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:07,398 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:07,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:08,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:11,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:12,610 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:12,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:16,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:21,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:21,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:23,703 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:11:25,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:25,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:25,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:25,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:25,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:26,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:27,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:27,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:30,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:31,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:32,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:37,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:42,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:42,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:44,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:44,729 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:44,729 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:44,729 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:45,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:46,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:47,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:47,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:47,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:50,624 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:52,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:52,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:53,704 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:11:57,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:58,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:59,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:59,612 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:59,612 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:59,613 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:59,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:02,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:02,629 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:03,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:04,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:08,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:08,860 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:13,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:13,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:17,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:17,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:17,560 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:17,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:17,561 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:17,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:18,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:19,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:22,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:23,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:23,705 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:12:24,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:28,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:29,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:32,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:34,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:38,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:38,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:38,334 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:38,334 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:38,335 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:38,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:40,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:40,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:42,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:44,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:45,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:47,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:50,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:53,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:53,706 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:12:55,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:56,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:56,627 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:56,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:56,628 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:56,649 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:58,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:58,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:00,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:00,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:02,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:02,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:05,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:08,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:10,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:11,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:11,508 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:11,509 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:11,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:11,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:12,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:14,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:16,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:16,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:17,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:21,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:23,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:23,707 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:13:26,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:27,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:27,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:27,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:27,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:28,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:28,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:30,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:32,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:32,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:32,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:37,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:37,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:42,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:42,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:45,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:45,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:45,436 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:45,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:45,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:46,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:47,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:48,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:48,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:50,672 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:52,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:53,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:53,708 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:13:57,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:58,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:02,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:03,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:05,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:05,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:05,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:05,845 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:06,678 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:07,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:08,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:08,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:10,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:13,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:13,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:18,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:18,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:23,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:23,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:23,709 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:14:25,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:25,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:25,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:25,451 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:25,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:26,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:28,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:29,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:30,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:33,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:33,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:38,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:38,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:43,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:43,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:43,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:43,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:43,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:43,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:44,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:44,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:48,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:48,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:49,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:53,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:53,710 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:14:55,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:58,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:00,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:01,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:01,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:01,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:01,291 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:01,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:02,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:03,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:03,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:05,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:06,703 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:09,011 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:10,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:14,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:15,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:17,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:17,727 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:17,727 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:17,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:18,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:18,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:19,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:19,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:19,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:20,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:22,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:23,711 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:15:24,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:26,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:29,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:31,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:34,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:36,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:37,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:37,658 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:37,658 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:37,658 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:37,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:38,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:40,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:41,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:42,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:45,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:46,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:49,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:51,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:53,712 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:15:55,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:55,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:55,639 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:55,639 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:55,639 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:55,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:56,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:57,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:00,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:00,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:02,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:04,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:07,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:10,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:12,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:15,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:16,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:16,166 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:16,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:16,167 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:16,729 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:18,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:18,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:19,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:20,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:23,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:23,713 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:16:24,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:28,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:29,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:30,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:30,016 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:30,016 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:30,016 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:30,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:32,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:34,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:34,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:34,736 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:39,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:39,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:44,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:44,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:47,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:47,263 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:47,263 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:47,263 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:47,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:48,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:49,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:49,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:49,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:52,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:53,713 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:16:54,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:54,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:59,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:00,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:03,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:03,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:03,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:03,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:04,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:04,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:04,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:05,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:05,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:05,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:08,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:10,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:10,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:15,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:15,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:20,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:20,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:23,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:23,566 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:23,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:23,567 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:23,714 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:17:23,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:24,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:25,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:26,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:28,757 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:30,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:31,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:35,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:35,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:38,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:38,039 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:38,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:38,040 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:38,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:38,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:40,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:41,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:42,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:45,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:46,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:50,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:51,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:53,715 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:17:55,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:56,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:57,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:57,147 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:57,148 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:57,148 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:57,768 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:58,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:00,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:02,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:02,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:05,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:07,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:10,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:12,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:15,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:17,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:19,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:19,614 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:19,614 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:19,614 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:19,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:20,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:21,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:22,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:23,716 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:18:24,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:26,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:27,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:31,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:32,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:35,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:35,948 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:35,948 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:35,948 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:36,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:36,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:36,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:37,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:40,785 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:41,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:43,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:46,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:48,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:50,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:50,558 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:50,559 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:50,559 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:50,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:51,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:52,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:53,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:53,722 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:18:54,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:57,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:58,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:02,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:03,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:06,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:08,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:10,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:10,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:10,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:10,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:11,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:12,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:12,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:13,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:14,798 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:17,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:19,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:23,718 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:19:24,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:26,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:27,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:27,495 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:27,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:27,497 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:27,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:28,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:30,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:32,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:32,805 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:35,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:36,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:40,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:41,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:41,102 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:41,103 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:41,103 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:41,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:42,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:42,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:44,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:46,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:47,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:51,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:51,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:53,719 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:19:56,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:56,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:01,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:01,519 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:01,519 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:01,521 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:01,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:01,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:02,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:02,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:06,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:06,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:07,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:12,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:12,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:17,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:17,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:21,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:22,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:22,109 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:22,109 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:22,110 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:22,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:22,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:23,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:23,720 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:20:26,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:26,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:28,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:31,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:33,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:36,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:38,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:39,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:39,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:39,133 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:39,135 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:39,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:40,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:41,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:42,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:43,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:46,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:48,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:51,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:53,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:53,721 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:20:56,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:58,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:00,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:00,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:00,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:00,112 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:00,843 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:00,844 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:01,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:04,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:04,845 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:06,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:09,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:11,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:14,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:16,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:16,996 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:16,996 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:16,998 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:17,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:18,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:20,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:20,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:21,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:23,722 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:21:25,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:27,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:30,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:32,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:35,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:36,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:38,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:38,434 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:38,434 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:38,435 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:38,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:40,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:41,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:42,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:42,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:46,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:47,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:51,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:51,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:53,723 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:21:55,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:55,461 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:55,461 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:55,462 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:55,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:56,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:57,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:57,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:00,868 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:02,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:02,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:06,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:07,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:12,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:12,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:14,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:14,486 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:14,486 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:14,487 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:14,873 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:16,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:17,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:18,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:18,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:21,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:23,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:23,724 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:22:27,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:28,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:32,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:32,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:32,942 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:32,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:32,943 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:33,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:33,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:34,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:36,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:36,886 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:39,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:41,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:44,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:46,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:49,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:51,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:53,725 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:22:54,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:56,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:57,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:57,092 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:57,092 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:57,092 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:57,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:59,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:00,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:01,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:01,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:05,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:06,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:10,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:10,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:10,742 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:10,742 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:10,743 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:10,899 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:12,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:13,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:15,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:15,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:17,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:20,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:21,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:23,725 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:23:26,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:26,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:31,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:31,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:32,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:32,520 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:32,520 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:32,520 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:32,907 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:33,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:36,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:37,698 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:37,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:42,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:42,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:47,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:47,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:47,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:47,799 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:47,799 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:47,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:47,913 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:49,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:51,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:52,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:53,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:55,332 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:23:56,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:58,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:01,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:02,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:02,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:02,777 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:02,777 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:02,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:03,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:05,920 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:06,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:07,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:08,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:11,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:13,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:16,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:18,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:21,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:22,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:22,833 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:22,833 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:22,834 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:22,927 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:23,727 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:24:24,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:25,928 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:26,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:27,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:30,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:31,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:35,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:36,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:40,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:41,567 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:42,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:42,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:42,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:42,653 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:42,934 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:43,935 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:45,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:47,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:47,936 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:50,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:51,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:53,728 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:24:56,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:56,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:01,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:01,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:02,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:02,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:02,600 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:02,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:02,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:03,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:06,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:06,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:07,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:11,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:12,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:16,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:17,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:21,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:21,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:22,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:22,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:22,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:22,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:22,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:23,729 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:25:23,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:26,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:27,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:27,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:31,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:32,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:36,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:37,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:41,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:41,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:41,907 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:41,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:41,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:41,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:42,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:43,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:47,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:47,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:48,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:51,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:53,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:53,730 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:25:56,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:58,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:01,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:03,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:03,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:03,394 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:03,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:03,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:04,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:05,965 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:06,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:07,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:09,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:12,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:14,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:17,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:19,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:21,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:23,731 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:26:24,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:24,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:24,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:24,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:24,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:25,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:25,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:26,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:29,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:30,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:31,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:35,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:36,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:40,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:41,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:44,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:44,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:44,338 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:44,338 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:44,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:45,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:46,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:47,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:49,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:51,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:51,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:53,732 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:26:56,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:57,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:01,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:02,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:02,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:02,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:02,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:02,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:03,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:06,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:07,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:07,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:11,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:12,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:16,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:17,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:21,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:21,024 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:21,024 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:21,025 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:21,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:21,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:23,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:23,733 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:27:23,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:25,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:26,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:28,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:31,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:33,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:36,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:38,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:41,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:42,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:42,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:42,529 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:42,529 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:43,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:44,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:47,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:48,002 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:48,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:51,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:53,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:53,733 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:27:57,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:58,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:02,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:03,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:05,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:05,932 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:05,933 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:05,933 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:06,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:06,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:08,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:08,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:11,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:12,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:14,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:16,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:19,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:21,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:23,735 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:28:24,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:26,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:26,598 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:26,598 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:26,599 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:26,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:27,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:28,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:30,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:32,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:32,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:35,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:36,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:40,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:42,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:45,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:45,116 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:45,116 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:45,116 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:46,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:46,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:47,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:48,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:50,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:51,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:51,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:53,736 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:28:56,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:56,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:01,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:01,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:04,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:04,879 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:04,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:04,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:05,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:06,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:06,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:06,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:10,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:11,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:12,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:16,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:17,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:21,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:22,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:23,737 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:29:24,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:24,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:24,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:24,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:25,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:26,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:27,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:27,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:30,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:32,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:32,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:36,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:37,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:42,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:42,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:43,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:43,557 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:43,557 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:43,558 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:44,045 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:46,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:47,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:47,703 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:48,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:51,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:52,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:53,737 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:29:57,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:57,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:02,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:02,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:07,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:09,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:09,779 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:09,779 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:09,779 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:10,055 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:12,056 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:12,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:12,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:14,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:17,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:17,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:21,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:22,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:23,738 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:30:27,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:28,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:30,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:30,284 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:30,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:30,285 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:31,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:32,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:33,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:34,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:36,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:39,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:42,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:44,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:47,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:49,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:51,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:53,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:53,192 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:53,193 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:53,193 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:53,739 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:30:54,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:54,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:56,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:56,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:58,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:00,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:01,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:05,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:06,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:10,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:10,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:10,517 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:10,517 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:10,517 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:11,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:12,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:12,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:15,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:16,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:17,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:20,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:23,740 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:31:26,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:27,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:30,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:30,827 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:30,827 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:30,828 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:31,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:31,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:32,089 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:32,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:36,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:36,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:42,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:46,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:47,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:51,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:51,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:52,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:52,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:52,337 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:52,337 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:53,096 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:53,741 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:31:54,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:57,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:57,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:58,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:02,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:02,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:07,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:12,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:12,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:15,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:15,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:15,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:15,314 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:16,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:16,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:17,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:18,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:20,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:23,742 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:32:27,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:28,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:29,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:29,474 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:29,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:31,337 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:32,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:32,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:32,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:34,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:36,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:39,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:42,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:44,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:47,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:48,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:48,719 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:48,719 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:48,720 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:49,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:49,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:50,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:51,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:53,742 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:32:54,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:55,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:57,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:00,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:02,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:05,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:08,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:08,975 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:08,975 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:08,975 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:09,124 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:10,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:11,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:11,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:14,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:16,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:16,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:21,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:21,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:23,744 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:33:26,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:26,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:29,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:29,245 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:29,246 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:29,246 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:30,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:30,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:31,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:32,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:34,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:36,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:37,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:41,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:42,419 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:46,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:47,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:51,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:51,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:51,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:51,345 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:51,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:52,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:52,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:53,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:53,744 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:33:56,144 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:57,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:58,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:02,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:03,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:06,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:07,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:07,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:07,979 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:07,979 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:08,148 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:09,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:10,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:11,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:12,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:14,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:16,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:19,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:21,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:22,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:22,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:22,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:22,283 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:23,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:24,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:24,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:25,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:34:27,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:28,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:29,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:32,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:34,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:36,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:39,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:39,769 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:39,769 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:39,770 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:40,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:40,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:42,162 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:42,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:44,163 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:45,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:47,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:50,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:51,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:53,746 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:34:56,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:56,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:56,838 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:56,838 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:56,840 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:57,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:57,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:58,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:02,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:02,170 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:02,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:06,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:07,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:12,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:12,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:15,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:15,389 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:15,389 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:15,390 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:16,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:16,176 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:17,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:18,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:20,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:21,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:22,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:23,747 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:35:27,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:28,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:32,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:33,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:34,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:34,382 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:34,382 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:34,384 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:35,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:36,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:36,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:38,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:39,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:42,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:44,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:47,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:49,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:51,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:52,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:52,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:52,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:52,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:52,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:53,748 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:35:54,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:54,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:56,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:57,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:00,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:02,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:05,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:06,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:06,817 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:06,818 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:06,820 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:06,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:07,195 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:08,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:11,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:12,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:12,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:16,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:17,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:21,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:21,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:23,749 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:36:26,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:26,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:26,365 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:26,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:26,367 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:27,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:28,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:28,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:30,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:31,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:33,153 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:36,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:36,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:41,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:42,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:44,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:44,761 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:44,761 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:44,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:45,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:46,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:46,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:47,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:50,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:51,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:51,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:53,751 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:36:57,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:57,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:58,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:58,635 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:58,635 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:58,635 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:59,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:00,218 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:03,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:03,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:04,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:06,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:08,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:12,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:13,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:15,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:15,499 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:15,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:15,501 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:16,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:16,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:18,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:18,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:20,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:21,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:23,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:23,751 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:37:27,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:28,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:32,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:33,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:34,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:34,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:34,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:34,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:35,230 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:36,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:37,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:37,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:38,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:40,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:42,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:43,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:47,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:48,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:51,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:51,795 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:51,796 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:51,798 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:52,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:52,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:52,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:53,753 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:37:54,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:56,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:57,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:00,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:02,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:05,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:07,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:10,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:11,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:11,352 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:11,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:11,353 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:12,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:12,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:13,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:15,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:16,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:18,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:20,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:22,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:23,754 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:38:25,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:26,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:26,099 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:26,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:26,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:26,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:27,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:28,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:30,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:31,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:32,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:36,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:37,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:41,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:42,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:43,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:43,954 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:43,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:43,954 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:44,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:46,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:46,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:48,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:48,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:52,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:52,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:53,755 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:38:57,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:57,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:01,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:01,341 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:01,341 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:01,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:02,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:02,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:03,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:03,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:06,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:07,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:08,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:13,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:13,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:17,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:17,146 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:17,146 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:17,147 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:17,269 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:18,269 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:18,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:19,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:22,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:22,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:23,756 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:39:24,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:27,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:29,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:32,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:33,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:33,870 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:33,870 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:33,871 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:34,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:34,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:34,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:37,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:40,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:40,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:45,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:47,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:50,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:52,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:52,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:52,750 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:52,750 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:52,751 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:53,283 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:53,757 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:39:54,283 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:55,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:56,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:58,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:00,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:03,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:05,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:07,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:09,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:09,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:09,130 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:09,130 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:09,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:10,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:11,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:12,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:15,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:16,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:17,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:21,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:22,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:23,758 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:40:25,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:25,180 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:25,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:25,181 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:25,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:27,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:27,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:27,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:31,299 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:32,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:32,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:37,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:37,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:40,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:40,878 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:40,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:40,878 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:41,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:42,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:42,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:43,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:45,304 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:47,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:47,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:48,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:52,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:53,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:53,759 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:40:57,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:57,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:57,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:57,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:57,680 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:58,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:58,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:59,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:03,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:03,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:04,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:07,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:09,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:12,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:14,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:17,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:17,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:17,512 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:17,512 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:17,512 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:18,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:19,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:19,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:22,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:23,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:23,760 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:41:24,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:28,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:29,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:33,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:34,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:35,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:35,976 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:35,976 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:35,977 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:36,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:37,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:37,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:40,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:41,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:42,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:45,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:47,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:50,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:52,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:53,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:53,496 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:53,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:53,497 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:53,762 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:41:54,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:55,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:55,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:58,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:59,333 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:01,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:03,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:06,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:07,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:11,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:13,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:14,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:14,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:14,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:14,904 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:15,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:16,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:17,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:18,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:19,340 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:22,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:22,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:23,762 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:42:27,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:28,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:32,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:32,632 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:32,633 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:32,633 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:33,346 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:33,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:34,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:35,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:37,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:37,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:38,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:43,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:43,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:48,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:48,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:52,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:53,763 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:42:54,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:54,187 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:54,187 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:54,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:54,188 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:54,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:55,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:57,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:59,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:59,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:02,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:04,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:07,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:09,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:12,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:14,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:14,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:14,387 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:14,388 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:14,388 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:15,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:17,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:18,103 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:19,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:19,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:22,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:23,764 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:43:24,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:28,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:29,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:33,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:34,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:36,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:36,311 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:36,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:36,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:36,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:37,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:38,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:40,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:41,372 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:43,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:45,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:49,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:50,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:53,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:53,765 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:43:55,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:57,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:57,667 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:57,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:57,667 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:58,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:58,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:59,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:00,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:03,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:03,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:05,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:08,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:10,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:13,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:15,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:18,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:20,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:23,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:23,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:23,312 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:23,312 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:23,312 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:23,388 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:23,766 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:44:25,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:26,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:27,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:28,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:29,390 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:31,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:33,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:36,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:38,019 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:40,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:40,079 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:40,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:40,080 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:40,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:41,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:42,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:43,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:45,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:47,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:48,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:52,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:53,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:53,767 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:44:54,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:54,416 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:54,416 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:54,416 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:55,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:55,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:57,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:59,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:59,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:02,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:04,153 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:07,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:08,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:12,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:14,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:16,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:16,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:16,966 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:16,967 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:17,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:17,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:19,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:19,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:21,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:23,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:23,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:23,767 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:45:28,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:28,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:33,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:33,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:38,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:38,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:40,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:40,447 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:40,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:40,448 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:41,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:41,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:44,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:44,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:45,423 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:49,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:49,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:53,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:53,769 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:45:54,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:58,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:58,369 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:58,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:58,370 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:58,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:59,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:59,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:00,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:03,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:05,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:05,567 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:08,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:10,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:14,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:15,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:19,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:20,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:21,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:21,783 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:21,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:21,784 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:22,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:23,438 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:23,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:23,770 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:46:26,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:27,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:28,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:31,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:33,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:36,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:38,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:39,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:39,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:39,966 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:39,966 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:40,445 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:41,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:42,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:43,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:45,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:47,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:48,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:52,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:53,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:53,770 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:46:57,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:57,487 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:57,487 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:57,487 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:57,488 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:58,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:59,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:59,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:01,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:02,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:03,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:04,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:07,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:08,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:12,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:14,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:17,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:19,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:19,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:47:19,330 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:47:19,330 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:47:19,330 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:47:19,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:47:21,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:23,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:23,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:23,772 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:47:28,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:29,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:33,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:34,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:38,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:38,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:40,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:47:40,787 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:47:40,788 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:47:40,788 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:47:41,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:47:43,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:43,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:43,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:45,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:48,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:49,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:53,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:53,773 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:47:54,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:58,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:59,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:03,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:04,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:05,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:05,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:05,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:05,669 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:06,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:07,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:08,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:09,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:09,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:11,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:14,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:14,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:19,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:19,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:23,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:23,773 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:48:24,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:24,985 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:24,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:24,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:24,986 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:25,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:27,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:28,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:29,489 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:30,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:33,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:35,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:38,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:39,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:39,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:39,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:39,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:39,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:40,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:41,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:43,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:44,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:45,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:49,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:50,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:53,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:53,774 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:48:54,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:54,252 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:54,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:54,253 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:54,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:55,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:56,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:59,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:59,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:01,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:04,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:06,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:08,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:11,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:14,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:16,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:19,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:19,065 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:19,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:19,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:19,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:19,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:21,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:22,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:23,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:23,775 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:49:27,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:28,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:32,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:33,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:35,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:35,908 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:35,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:35,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:36,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:37,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:37,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:38,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:41,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:43,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:43,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:48,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:48,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:51,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:51,002 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:51,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:51,003 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:51,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:53,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:53,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:53,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:53,777 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:49:53,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:55,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:58,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:59,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:03,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:04,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:08,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:09,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:12,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:12,034 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:12,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:12,035 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:12,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:13,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:13,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:15,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:17,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:18,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:20,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:23,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:23,778 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:50:25,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:28,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:30,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:31,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:31,977 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:31,977 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:31,978 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:32,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:33,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:34,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:36,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:37,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:38,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:41,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:43,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:46,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:48,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:50,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:50,425 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:50,425 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:50,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:50,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:51,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:51,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:53,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:53,779 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:50:55,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:59,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:01,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:04,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:06,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:06,208 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:06,208 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:06,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:06,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:07,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:07,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:08,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:11,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:12,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:13,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:17,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:18,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:22,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:23,780 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:51:23,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:24,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:24,134 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:24,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:24,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:24,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:25,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:27,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:29,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:29,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:32,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:34,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:37,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:38,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:39,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:39,828 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:39,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:39,829 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:40,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:41,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:42,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:43,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:44,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:48,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:49,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:53,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:53,780 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:51:53,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:54,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:54,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:54,528 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:54,528 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:54,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:55,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:58,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:59,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:59,574 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:03,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:04,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:08,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:08,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:11,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:11,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:11,456 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:11,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:11,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:13,580 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:13,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:14,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:15,581 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:19,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:19,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:23,782 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:52:23,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:24,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:28,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:28,493 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:28,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:28,494 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:28,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:29,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:29,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:30,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:33,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:34,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:36,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:38,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:41,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:44,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:46,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:49,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:51,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:52,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:52,876 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:52,877 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:52,877 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:53,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:53,596 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:53,783 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:52:54,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:54,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:54,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:56,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:57,597 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:59,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:01,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:04,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:06,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:09,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:09,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:11,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:12,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:12,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:12,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:12,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:13,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:13,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:15,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:16,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:17,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:20,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:21,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:23,784 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:53:24,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:26,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:29,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:30,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:30,673 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:30,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:30,674 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:31,611 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:31,611 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:32,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:35,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:35,612 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:37,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:39,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:42,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:45,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:47,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:50,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:51,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:51,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:51,510 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:51,511 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:51,618 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:53,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:53,619 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:53,785 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:53:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:55,620 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:58,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:00,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:03,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:05,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:08,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:09,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:11,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:54:11,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:54:11,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:54:11,484 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:54:11,625 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:54:13,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:14,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:15,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:17,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:19,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:20,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:23,785 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:54:24,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:25,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:30,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:30,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:33,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:54:33,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:54:33,046 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:54:33,047 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:54:33,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:54:33,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:35,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:36,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:37,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:39,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:41,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:44,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:46,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:49,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:51,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:53,786 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:54:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:56,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:59,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:01,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:01,463 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:01,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:01,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:01,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:02,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:03,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:05,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:05,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:07,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:09,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:12,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:15,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:17,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:20,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:20,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:20,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:20,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:20,856 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:21,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:21,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:22,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:23,787 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:55:24,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:25,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:28,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:30,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:33,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:35,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:36,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:36,460 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:36,460 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:36,461 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:36,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:37,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:38,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:39,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:41,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:43,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:45,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:48,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:50,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:52,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:52,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:52,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:52,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:53,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:53,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:53,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:53,788 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:55:55,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:55,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:57,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:58,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:00,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:03,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:05,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:06,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:06,473 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:06,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:06,474 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:06,666 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:07,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:09,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:10,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:11,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:14,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:16,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:19,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:21,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:23,789 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:56:23,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:23,909 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:23,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:23,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:24,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:24,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:25,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:25,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:27,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:30,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:30,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:35,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:35,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:40,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:40,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:42,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:42,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:42,510 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:42,510 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:42,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:43,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:45,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:45,681 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:46,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:47,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:50,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:51,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:53,790 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:56:55,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:55,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:59,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:59,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:59,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:59,992 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:00,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:01,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:01,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:01,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:03,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:06,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:06,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:10,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:11,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:15,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:16,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:20,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:20,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:20,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:20,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:20,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:21,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:21,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:21,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:23,791 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:57:25,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:25,697 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:26,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:31,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:31,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:36,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:36,436 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:36,437 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:36,437 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:36,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:36,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:37,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:37,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:39,702 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:40,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:42,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:46,173 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:47,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:51,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:52,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:53,792 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:57:55,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:55,146 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:55,146 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:55,146 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:55,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:55,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:55,709 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:58,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:59,710 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:00,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:03,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:05,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:08,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:10,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:13,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:15,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:16,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:16,029 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:16,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:16,030 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:16,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:17,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:19,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:19,719 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:23,793 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:58:24,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:25,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:29,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:30,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:30,958 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:30,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:30,959 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:31,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:31,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:32,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:35,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:36,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:36,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:40,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:40,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:45,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:45,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:50,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:50,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:50,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:50,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:50,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:50,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:50,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:52,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:53,794 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:58:54,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:55,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:55,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:00,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:01,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:05,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:06,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:07,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:07,492 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:07,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:07,493 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:07,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:08,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:10,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:11,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:12,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:16,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:16,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:21,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:21,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:23,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:23,438 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:23,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:23,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:23,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:23,794 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:59:24,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:26,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:27,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:28,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:31,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:32,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:36,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:37,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:41,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:41,021 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:41,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:41,022 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:41,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:41,752 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:42,753 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:43,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:46,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:46,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:48,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:51,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:53,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:53,795 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:59:56,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:58,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:01,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:01,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:01,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:01,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:01,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:01,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:02,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:04,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:06,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:07,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:09,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:11,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:14,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:16,265 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:19,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:21,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:23,796 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:00:24,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:26,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:26,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:26,959 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:26,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:26,960 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:27,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:28,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:29,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:32,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:35,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:36,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:40,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:41,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:45,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:46,695 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:50,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:50,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:50,562 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:50,562 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:50,563 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:50,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:52,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:52,779 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:53,797 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:00:54,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:55,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:56,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:00,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:01,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:05,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:06,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:10,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:10,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:10,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:10,980 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:10,980 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:11,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:11,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:12,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:16,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:16,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:16,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:21,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:21,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:23,797 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:01:26,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:26,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:31,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:31,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:32,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:32,497 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:32,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:32,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:32,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:34,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:37,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:37,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:38,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:41,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:42,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:46,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:47,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:50,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:50,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:50,165 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:50,165 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:50,800 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:51,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:52,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:53,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:53,799 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:01:54,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:56,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:58,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:01,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:03,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:06,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:06,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:06,961 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:06,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:06,961 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:07,806 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:08,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:08,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:11,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:12,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:14,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:16,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:19,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:21,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:23,799 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:02:24,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:24,273 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:24,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:24,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:24,275 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:24,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:26,813 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:26,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:26,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:28,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:29,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:31,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:34,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:36,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:39,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:41,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:41,511 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:41,511 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:41,511 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:41,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:41,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:42,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:44,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:46,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:47,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:49,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:52,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:53,800 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:02:55,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:56,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:59,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:59,575 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:59,576 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:59,576 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:59,827 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:01,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:02,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:02,828 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:04,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:06,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:07,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:11,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:11,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:16,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:17,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:17,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:17,724 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:17,724 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:17,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:17,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:20,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:22,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:22,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:22,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:23,801 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:03:26,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:27,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:32,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:32,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:37,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:37,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:38,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:38,606 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:38,606 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:38,607 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:38,841 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:40,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:41,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:42,843 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:43,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:44,844 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:47,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:48,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:52,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:53,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:53,418 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:53,418 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:53,419 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:53,802 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:03:53,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:53,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:54,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:56,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:58,850 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:59,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:02,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:04,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:07,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:09,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:11,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:12,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:12,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:12,799 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:12,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:12,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:14,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:14,856 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:17,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:18,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:19,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:22,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:23,803 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:04:25,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:26,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:28,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:28,563 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:28,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:28,564 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:28,861 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:30,862 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:31,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:32,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:32,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:34,864 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:36,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:37,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:41,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:41,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:46,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:47,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:48,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:48,534 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:48,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:48,869 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:49,256 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:50,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:52,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:52,871 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:53,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:53,804 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:04:54,872 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:56,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:57,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:02,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:02,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:07,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:07,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:11,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:12,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:13,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:13,879 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:13,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:13,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:13,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:16,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:17,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:18,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:18,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:22,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:23,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:23,804 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:05:26,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:28,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:32,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:32,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:32,652 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:32,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:32,652 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:32,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:34,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:34,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:36,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:38,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:39,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:41,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:44,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:47,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:47,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:47,411 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:47,411 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:47,411 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:47,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:48,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:50,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:52,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:53,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:53,805 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:05:55,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:56,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:00,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:02,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:05,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:05,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:05,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:05,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:05,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:06,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:06,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:07,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:10,903 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:11,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:11,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:16,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:17,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:21,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:22,422 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:23,806 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:06:26,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:26,962 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:26,963 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:26,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:26,964 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:27,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:28,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:32,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:32,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:32,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:37,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:37,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:41,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:42,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:46,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:46,176 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:46,176 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:46,177 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:46,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:47,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:47,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:48,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:50,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:52,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:52,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:53,807 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:06:56,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:57,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:01,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:02,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:06,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:07,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:07,506 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:07,506 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:07,506 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:07,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:08,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:08,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:11,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:12,927 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:13,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:17,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:18,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:22,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:23,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:23,808 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:07:26,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:26,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:26,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:26,821 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:26,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:26,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:28,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:28,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:30,934 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:32,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:33,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:37,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:38,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:41,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:43,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:47,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:48,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:49,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:49,892 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:49,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:49,892 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:49,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:50,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:52,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:53,809 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:07:54,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:54,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:56,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:59,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:02,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:04,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:07,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:09,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:10,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:10,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:10,508 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:10,508 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:10,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:12,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:12,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:14,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:15,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:17,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:20,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:22,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:23,810 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:08:25,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:26,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:26,003 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:26,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:26,004 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:26,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:26,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:27,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:27,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:27,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:30,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:31,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:32,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:36,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:37,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:41,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:42,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:46,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:47,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:49,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:49,846 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:49,846 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:49,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:49,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:50,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:51,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:53,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:53,811 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:08:54,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:57,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:57,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:02,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:03,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:06,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:06,173 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:06,173 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:06,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:06,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:08,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:08,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:10,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:12,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:13,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:17,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:18,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:23,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:23,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:23,816 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:09:27,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:27,236 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:27,237 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:27,237 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:27,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:27,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:28,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:29,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:32,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:32,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:34,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:37,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:39,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:42,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:42,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:42,867 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:42,868 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:42,868 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:42,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:44,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:44,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:46,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:48,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:50,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:53,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:53,813 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:09:55,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:57,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:00,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:03,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:05,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:05,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:05,540 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:05,542 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:05,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:06,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:06,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:09,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:10,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:11,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:12,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:16,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:18,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:21,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:22,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:22,487 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:22,487 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:22,487 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:23,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:23,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:23,813 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:10:25,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:27,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:27,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:27,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:32,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:33,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:37,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:38,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:38,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:38,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:38,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:38,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:39,005 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:41,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:42,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:43,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:43,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:48,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:48,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:53,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:53,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:53,815 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:10:57,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:58,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:03,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:03,523 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:03,524 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:03,524 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:03,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:04,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:04,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:05,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:09,017 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:09,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:09,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:12,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:14,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:18,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:19,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:22,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:22,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:22,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:22,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:23,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:23,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:23,816 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:11:24,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:25,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:27,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:27,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:30,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:33,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:35,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:38,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:38,542 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:38,542 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:38,542 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:38,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:39,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:40,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:41,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:42,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:43,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:46,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:48,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:51,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:53,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:53,817 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:11:56,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:57,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:58,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:58,642 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:58,642 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:58,643 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:59,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:01,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:01,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:03,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:03,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:06,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:08,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:11,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:12,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:16,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:12:16,901 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:12:16,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:12:16,902 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:12:17,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:18,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:19,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:21,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:22,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:23,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:23,818 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:12:27,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:27,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:32,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:33,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:37,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:38,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:40,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:12:40,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:12:40,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:12:40,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:12:41,060 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:43,061 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:43,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:43,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:45,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:48,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:48,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:53,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:53,820 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:12:54,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:58,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:59,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:03,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:04,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:08,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:08,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:08,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:08,514 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:08,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:09,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:10,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:11,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:13,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:13,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:15,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:19,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:20,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:23,820 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:13:24,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:25,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:28,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:30,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:33,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:33,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:33,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:33,723 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:34,080 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:34,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:35,080 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:36,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:39,082 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:39,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:43,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:46,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:48,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:48,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:48,903 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:48,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:48,904 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:49,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:51,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:51,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:53,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:53,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:53,821 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:13:57,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:58,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:02,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:03,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:07,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:08,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:12,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:12,114 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:12,114 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:12,115 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:13,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:13,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:13,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:13,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:13,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:17,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:18,294 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:18,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:23,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:23,822 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:14:23,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:28,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:28,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:30,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:30,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:30,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:30,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:30,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:31,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:34,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:34,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:35,103 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:39,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:39,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:43,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:44,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:44,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:44,920 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:44,921 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:44,921 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:45,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:47,108 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:49,109 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:49,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:50,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:53,823 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:14:54,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:55,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:58,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:00,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:03,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:03,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:03,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:03,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:04,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:04,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:05,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:06,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:09,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:10,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:11,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:13,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:16,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:18,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:18,413 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:18,414 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:18,414 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:19,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:19,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:20,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:22,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:23,123 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:23,824 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:15:25,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:27,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:28,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:32,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:34,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:36,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:36,466 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:36,466 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:36,466 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:37,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:37,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:38,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:39,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:41,129 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:43,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:43,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:48,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:49,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:53,825 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:15:54,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:54,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:54,631 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:54,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:54,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:54,632 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:55,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:55,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:58,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:59,137 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:59,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:04,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:04,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:09,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:09,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:11,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:11,716 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:11,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:11,717 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:12,141 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:13,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:14,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:14,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:17,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:19,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:19,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:23,826 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:16:24,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:25,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:29,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:30,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:31,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:31,566 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:31,567 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:31,567 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:32,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:34,150 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:35,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:35,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:36,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:38,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:40,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:40,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:44,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:45,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:49,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:50,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:53,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:53,527 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:53,527 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:53,528 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:53,827 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:16:54,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:55,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:56,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:56,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:58,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:59,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:01,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:05,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:06,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:11,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:14,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:15,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:15,579 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:15,579 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:15,580 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:16,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:17,618 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:18,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:20,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:20,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:22,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:23,828 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:17:25,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:27,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:32,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:32,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:32,314 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:32,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:33,173 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:33,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:34,173 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:35,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:38,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:38,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:40,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:43,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:48,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:50,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:52,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:52,559 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:52,560 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:52,560 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:53,181 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:53,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:53,829 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:17:54,182 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:58,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:58,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:59,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:03,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:05,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:08,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:10,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:13,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:14,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:17,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:17,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:17,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:17,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:18,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:19,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:20,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:20,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:22,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:23,830 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:18:24,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:25,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:29,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:34,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:35,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:36,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:36,127 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:36,127 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:36,128 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:36,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:38,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:40,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:40,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:42,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:45,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:49,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:50,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:53,830 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:18:54,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:55,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:56,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:56,251 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:56,251 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:56,252 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:57,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:58,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:59,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:01,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:02,206 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:05,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:06,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:10,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:11,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:11,077 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:11,077 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:11,078 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:11,210 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:12,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:14,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:14,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:16,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:17,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:19,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:22,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:23,832 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:19:24,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:27,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:32,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:34,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:36,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:36,331 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:36,331 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:36,331 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:37,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:38,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:38,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:40,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:42,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:43,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:44,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:48,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:50,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:53,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:53,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:19:54,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:54,433 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:54,433 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:54,434 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:55,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:55,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:56,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:58,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:59,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:00,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:03,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:05,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:08,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:10,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:13,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:14,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:17,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:17,142 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:17,142 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:17,143 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:17,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:19,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:19,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:20,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:22,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:23,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:20:24,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:24,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:29,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:34,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:34,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:37,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:37,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:37,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:37,222 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:37,241 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:39,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:40,242 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:40,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:42,243 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:44,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:45,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:49,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:50,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:53,834 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:20:54,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:55,353 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:55,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:55,354 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:56,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:56,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:58,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:59,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:00,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:01,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:05,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:06,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:10,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:11,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:14,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:16,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:20,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:20,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:20,369 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:20,370 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:20,370 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:21,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:22,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:22,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:23,836 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:21:26,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:26,260 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:27,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:29,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:32,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:35,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:37,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:37,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:37,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:37,313 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:38,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:38,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:40,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:40,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:42,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:43,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:44,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:48,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:50,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:53,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:53,837 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:21:55,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:57,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:57,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:57,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:57,219 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:57,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:59,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:59,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:00,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:02,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:04,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:04,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:09,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:09,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:14,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:14,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:16,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:16,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:16,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:16,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:16,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:18,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:19,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:20,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:22,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:23,838 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:22:24,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:25,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:29,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:30,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:32,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:32,910 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:32,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:32,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:33,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:34,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:35,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:35,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:38,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:40,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:41,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:44,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:46,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:47,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:47,941 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:47,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:48,828 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:49,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:50,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:51,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:51,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:53,839 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:22:54,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:56,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:57,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:59,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:02,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:05,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:07,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:08,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:08,636 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:08,636 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:08,637 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:09,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:10,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:11,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:12,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:14,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:14,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:17,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:20,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:22,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:22,675 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:22,675 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:22,675 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:23,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:23,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:23,839 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:23:24,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:25,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:28,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:28,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:29,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:33,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:35,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:37,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:37,401 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:37,402 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:37,402 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:38,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:39,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:40,313 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:41,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:42,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:44,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:44,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:49,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:50,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:53,840 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:23:54,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:54,157 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:54,157 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:54,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:54,318 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:55,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:55,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:56,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:58,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:59,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:00,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:04,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:05,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:08,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:08,531 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:08,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:08,534 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:09,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:10,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:10,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:10,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:14,328 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:14,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:15,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:20,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:20,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:23,841 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:24:25,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:25,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:29,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:30,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:31,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:31,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:31,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:31,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:32,334 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:34,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:35,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:36,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:36,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:40,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:41,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:44,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:46,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:50,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:51,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:53,842 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:24:54,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:54,922 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:54,922 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:54,923 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:55,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:55,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:56,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:56,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:59,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:00,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:02,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:05,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:07,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:10,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:12,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:15,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:15,865 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:15,866 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:15,866 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:16,351 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:17,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:18,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:20,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:20,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:23,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:23,843 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:25:25,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:28,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:29,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:31,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:31,112 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:31,112 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:31,112 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:31,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:32,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:34,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:34,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:36,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:39,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:39,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:44,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:44,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:48,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:48,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:48,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:48,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:48,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:49,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:50,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:50,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:52,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:53,844 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:25:54,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:55,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:00,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:04,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:05,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:07,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:07,032 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:07,032 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:07,033 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:07,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:08,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:10,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:11,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:12,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:16,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:20,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:21,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:23,845 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:26:25,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:26,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:29,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:29,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:29,944 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:29,944 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:29,944 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:30,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:31,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:32,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:34,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:35,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:37,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:40,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:42,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:44,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:46,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:46,604 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:46,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:46,604 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:47,393 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:47,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:48,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:50,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:52,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:52,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:53,846 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:26:55,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:57,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:02,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:05,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:06,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:06,175 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:06,176 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:06,176 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:06,400 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:08,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:08,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:10,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:10,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:13,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:14,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:18,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:19,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:23,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:23,846 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:27:24,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:25,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:25,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:25,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:25,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:26,408 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:28,409 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:28,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:29,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:30,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:34,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:35,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:39,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:40,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:42,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:42,621 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:42,622 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:42,622 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:43,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:44,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:44,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:45,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:46,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:48,416 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:49,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:50,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:53,848 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:27:54,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:55,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:59,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:00,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:01,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:01,079 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:01,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:01,569 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:02,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:02,423 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:05,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:05,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:06,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:10,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:15,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:15,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:19,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:19,381 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:19,382 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:19,382 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:19,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:20,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:20,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:21,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:23,848 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:28:24,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:25,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:26,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:30,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:30,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:35,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:36,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:39,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:39,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:39,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:39,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:39,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:40,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:41,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:41,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:44,439 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:45,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:46,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:50,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:51,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:53,849 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:28:55,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:56,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:58,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:58,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:58,250 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:58,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:58,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:00,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:00,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:02,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:02,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:05,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:07,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:10,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:12,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:14,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:14,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:14,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:14,549 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:15,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:16,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:16,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:16,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:17,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:18,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:21,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:22,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:23,851 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:29:26,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:27,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:31,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:32,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:36,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:37,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:37,368 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:37,368 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:37,369 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:37,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:38,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:38,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:42,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:42,464 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:43,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:46,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:48,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:52,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:53,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:53,851 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:29:54,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:54,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:54,343 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:54,343 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:54,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:56,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:57,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:58,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:59,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:01,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:04,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:07,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:09,514 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:12,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:14,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:16,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:16,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:16,924 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:16,924 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:16,925 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:17,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:18,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:19,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:21,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:22,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:23,852 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:30:25,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:26,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:30,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:31,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:33,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:33,172 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:33,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:33,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:33,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:34,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:36,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:36,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:38,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:41,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:41,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:46,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:46,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:50,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:50,151 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:50,151 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:50,151 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:50,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:51,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:52,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:52,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:53,853 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:30:54,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:56,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:57,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:01,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:02,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:06,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:07,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:10,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:10,495 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:10,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:10,496 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:10,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:12,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:12,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:13,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:14,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:16,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:18,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:22,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:23,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:23,855 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:31:27,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:28,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:28,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:28,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:28,553 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:29,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:29,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:30,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:31,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:32,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:34,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:37,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:39,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:42,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:44,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:44,698 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:44,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:44,699 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:45,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:45,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:46,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:46,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:48,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:50,515 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:51,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:51,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:53,856 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:31:56,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:56,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:01,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:06,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:06,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:10,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:10,251 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:10,251 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:10,251 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:10,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:11,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:12,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:12,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:14,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:16,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:17,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:21,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:22,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:23,856 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:32:26,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:27,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:31,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:31,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:31,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:31,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:31,993 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:32,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:32,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:33,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:36,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:36,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:38,197 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:41,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:43,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:46,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:47,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:47,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:47,427 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:47,428 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:47,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:48,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:48,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:52,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:52,537 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:53,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:53,857 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:32:57,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:58,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:03,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:04,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:04,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:04,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:04,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:05,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:06,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:06,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:08,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:08,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:11,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:13,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:16,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:18,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:21,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:23,859 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:33:24,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:26,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:28,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:28,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:28,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:28,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:28,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:30,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:30,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:31,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:32,551 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:35,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:37,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:40,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:42,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:45,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:46,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:49,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:49,529 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:49,529 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:49,529 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:49,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:50,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:50,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:52,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:53,859 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:33:54,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:55,860 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:57,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:00,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:01,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:05,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:07,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:08,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:08,923 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:08,924 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:08,924 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:09,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:10,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:10,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:12,565 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:12,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:14,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:16,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:16,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:21,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:21,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:23,860 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:34:26,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:26,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:29,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:29,562 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:29,563 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:29,563 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:29,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:31,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:32,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:32,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:32,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:35,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:37,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:37,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:42,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:42,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:47,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:47,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:52,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:52,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:52,378 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:52,378 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:52,379 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:52,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:53,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:53,867 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:34:55,583 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:57,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:58,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:58,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:02,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:03,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:08,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:08,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:13,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:13,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:17,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:18,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:19,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:19,022 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:19,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:19,023 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:19,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:35:21,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:22,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:23,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:23,863 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:35:23,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:27,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:28,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:32,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:33,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:37,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:38,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:38,581 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:38,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:38,583 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:38,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:35:39,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:41,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:43,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:43,602 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:44,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:47,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:49,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:52,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:53,864 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:35:54,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:57,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:58,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:58,687 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:58,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:58,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:59,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:00,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:01,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:02,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:03,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:05,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:07,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:10,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:12,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:15,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:17,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:18,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:18,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:18,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:18,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:18,614 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:21,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:21,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:22,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:23,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:23,865 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:36:26,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:27,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:31,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:32,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:36,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:36,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:36,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:36,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:37,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:37,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:37,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:39,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:41,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:42,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:42,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:47,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:47,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:52,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:52,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:53,866 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:36:54,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:54,967 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:54,968 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:54,968 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:55,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:57,629 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:58,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:58,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:59,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:02,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:03,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:07,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:08,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:11,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:11,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:11,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:11,450 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:11,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:13,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:13,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:13,636 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:15,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:17,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:17,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:18,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:23,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:23,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:23,867 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:37:28,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:28,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:32,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:33,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:34,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:34,121 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:34,121 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:34,121 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:34,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:35,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:37,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:39,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:39,647 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:42,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:44,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:47,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:49,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:49,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:49,959 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:49,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:49,960 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:50,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:51,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:52,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:53,868 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:37:54,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:55,654 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:57,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:59,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:02,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:04,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:06,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:06,686 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:06,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:06,687 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:07,658 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:08,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:09,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:10,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:11,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:13,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:15,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:17,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:20,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:21,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:21,994 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:21,994 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:21,994 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:22,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:22,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:23,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:23,869 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:38:26,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:27,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:27,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:31,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:32,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:36,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:37,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:38,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:38,590 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:38,590 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:38,591 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:38,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:39,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:41,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:43,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:43,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:46,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:47,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:51,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:52,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:53,870 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:38:56,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:57,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:58,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:58,157 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:58,157 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:58,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:58,696 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:59,697 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:02,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:02,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:03,698 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:07,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:08,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:12,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:13,112 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:17,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:17,138 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:17,138 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:17,138 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:17,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:17,703 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:18,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:19,704 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:21,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:22,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:23,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:23,871 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:39:27,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:28,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:32,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:33,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:36,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:36,230 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:36,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:36,230 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:36,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:37,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:37,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:39,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:41,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:42,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:44,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:47,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:49,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:51,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:51,601 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:51,602 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:51,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:51,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:53,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:53,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:54,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:56,283 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:39:57,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:58,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:00,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:02,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:05,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:07,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:10,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:12,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:15,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:18,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:18,800 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:18,800 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:18,801 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:19,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:19,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:20,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:22,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:23,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:23,873 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:40:25,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:27,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:30,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:32,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:35,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:37,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:38,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:38,297 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:38,297 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:38,299 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:38,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:39,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:41,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:43,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:43,737 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:47,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:51,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:53,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:53,873 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:40:53,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:53,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:53,980 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:53,980 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:54,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:55,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:57,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:58,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:59,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:02,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:02,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:07,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:07,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:12,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:12,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:13,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:13,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:13,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:13,541 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:13,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:15,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:17,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:17,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:22,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:22,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:23,874 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:41:27,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:27,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:32,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:32,117 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:32,117 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:32,118 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:32,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:32,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:33,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:33,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:37,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:37,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:38,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:42,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:43,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:47,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:48,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:52,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:53,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:53,875 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:41:54,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:54,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:54,435 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:54,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:54,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:55,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:58,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:58,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:59,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:02,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:03,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:08,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:08,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:13,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:13,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:14,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:14,326 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:14,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:14,327 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:14,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:15,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:19,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:19,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:23,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:23,876 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:42:24,877 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:28,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:29,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:31,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:31,881 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:31,881 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:31,882 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:32,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:32,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:33,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:34,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:35,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:37,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:37,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:40,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:42,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:45,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:45,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:45,601 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:45,601 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:45,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:45,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:47,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:47,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:49,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:50,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:52,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:53,878 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:42:55,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:57,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:59,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:59,697 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:59,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:59,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:59,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:01,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:01,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:02,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:05,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:07,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:07,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:12,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:12,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:17,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:17,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:20,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:20,167 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:20,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:20,167 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:20,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:21,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:22,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:23,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:23,879 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:43:25,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:27,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:28,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:32,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:33,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:37,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:37,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:37,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:37,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:37,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:37,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:39,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:39,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:41,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:43,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:44,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:47,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:49,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:53,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:53,880 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:43:54,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:54,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:54,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:54,179 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:54,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:55,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:55,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:58,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:59,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:00,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:02,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:05,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:07,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:10,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:12,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:13,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:13,858 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:13,858 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:13,859 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:14,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:15,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:15,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:17,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:17,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:21,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:22,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:23,880 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:44:26,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:27,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:27,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:27,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:27,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:27,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:27,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:29,821 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:31,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:31,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:32,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:36,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:37,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:41,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:42,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:46,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:47,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:50,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:50,931 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:50,931 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:50,932 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:51,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:51,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:51,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:52,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:53,886 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:44:55,830 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:57,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:57,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:02,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:02,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:07,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:07,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:12,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:12,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:12,710 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:12,711 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:12,711 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:12,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:12,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:13,837 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:17,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:17,839 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:17,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:22,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:22,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:23,882 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:45:27,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:27,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:32,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:32,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:35,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:35,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:35,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:35,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:35,846 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:37,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:38,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:38,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:39,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:43,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:43,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:47,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:48,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:52,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:53,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:53,883 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:45:53,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:53,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:53,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:53,956 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:54,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:55,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:57,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:57,856 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:59,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:02,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:04,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:07,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:09,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:10,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:10,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:10,428 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:10,430 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:10,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:11,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:13,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:14,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:15,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:17,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:19,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:23,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:23,884 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:46:25,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:28,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:30,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:32,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:35,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:35,124 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:35,124 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:35,125 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:35,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:35,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:36,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:37,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:39,876 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:41,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:42,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:46,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:47,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:51,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:53,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:53,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:53,686 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:53,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:53,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:53,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:53,885 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:46:55,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:56,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:57,884 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:58,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:01,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:02,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:06,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:07,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:07,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:07,965 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:07,965 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:07,966 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:08,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:09,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:11,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:12,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:12,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:17,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:22,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:22,097 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:22,098 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:22,098 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:22,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:22,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:23,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:23,886 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:47:23,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:27,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:27,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:28,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:32,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:33,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:37,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:38,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:41,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:41,494 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:41,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:41,495 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:41,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:43,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:43,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:43,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:45,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:47,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:48,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:52,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:53,887 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:47:54,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:57,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:59,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:02,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:04,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:05,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:05,409 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:05,409 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:05,410 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:05,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:05,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:08,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:09,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:10,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:13,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:15,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:17,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:20,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:23,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:23,888 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:48:24,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:24,239 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:24,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:24,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:24,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:25,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:26,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:27,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:28,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:31,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:32,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:36,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:37,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:41,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:42,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:42,253 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:42,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:42,254 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:42,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:42,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:43,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:45,926 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:47,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:51,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:52,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:53,889 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:48:56,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:56,890 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:56,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:56,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:56,891 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:56,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:57,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:58,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:01,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:02,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:02,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:07,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:07,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:12,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:12,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:15,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:15,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:15,937 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:15,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:17,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:17,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:17,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:19,939 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:22,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:23,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:23,890 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:49:27,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:28,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:32,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:32,616 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:32,616 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:32,617 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:32,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:32,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:33,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:33,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:37,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:38,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:38,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:43,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:43,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:47,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:48,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:50,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:50,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:50,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:50,540 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:50,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:51,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:53,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:53,891 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:49:53,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:53,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:55,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:58,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:58,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:02,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:03,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:07,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:07,069 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:07,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:07,070 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:07,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:07,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:08,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:09,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:11,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:13,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:14,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:17,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:19,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:23,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:23,892 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:50:24,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:26,946 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:26,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:26,946 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:26,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:27,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:29,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:29,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:31,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:32,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:35,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:38,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:40,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:43,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:45,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:46,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:46,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:46,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:46,857 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:46,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:47,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:48,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:48,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:51,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:51,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:53,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:53,893 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:50:56,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:58,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:01,895 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:03,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:04,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:04,912 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:04,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:04,912 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:04,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:05,980 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:06,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:08,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:09,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:12,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:13,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:17,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:18,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:22,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:23,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:23,894 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:51:24,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:24,102 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:24,102 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:24,103 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:24,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:25,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:27,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:27,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:28,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:32,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:33,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:37,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:38,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:42,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:43,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:47,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:48,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:51,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:51,231 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:51,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:51,232 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:51,997 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:51,998 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:53,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:53,895 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:51:53,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:55,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:58,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:58,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:03,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:03,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:08,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:08,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:09,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:09,802 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:09,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:09,803 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:10,004 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:10,004 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:13,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:14,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:14,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:18,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:18,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:23,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:23,896 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:52:24,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:26,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:26,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:26,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:26,729 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:27,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:29,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:29,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:29,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:31,012 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:33,013 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:33,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:34,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:39,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:39,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:41,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:41,428 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:41,429 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:41,429 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:42,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:43,017 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:45,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:45,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:47,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:48,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:50,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:53,897 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:52:54,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:55,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:59,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:00,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:01,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:01,173 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:01,174 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:01,174 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:02,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:03,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:03,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:06,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:07,027 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:08,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:11,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:13,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:16,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:18,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:19,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:19,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:19,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:19,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:20,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:21,033 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:21,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:23,897 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:53:24,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:25,034 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:26,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:29,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:31,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:33,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:36,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:37,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:37,505 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:37,505 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:37,506 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:38,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:39,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:39,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:42,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:43,042 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:44,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:47,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:48,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:52,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:52,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:52,860 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:52,860 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:52,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:53,045 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:53,898 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:53:54,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:55,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:57,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:59,048 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:59,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:03,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:03,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:08,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:09,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:13,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:13,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:13,595 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:13,595 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:13,595 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:14,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:15,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:15,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:18,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:18,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:19,055 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:23,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:23,899 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:54:24,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:28,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:29,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:33,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:33,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:36,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:36,191 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:36,191 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:36,192 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:37,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:38,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:39,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:39,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:41,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:43,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:44,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:48,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:49,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:51,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:51,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:51,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:51,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:51,067 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:53,068 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:53,905 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:54:54,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:55,069 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:55,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:57,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:59,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:00,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:05,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:05,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:05,943 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:05,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:05,943 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:06,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:07,074 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:09,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:10,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:11,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:14,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:15,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:18,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:20,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:23,901 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:55:24,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:25,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:25,611 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:25,611 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:25,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:25,613 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:26,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:27,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:30,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:30,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:31,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:33,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:35,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:39,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:40,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:40,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:40,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:40,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:41,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:41,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:43,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:45,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:45,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:46,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:48,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:51,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:53,902 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:55:54,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:56,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:57,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:57,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:57,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:57,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:58,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:59,094 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:59,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:02,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:03,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:03,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:07,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:09,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:12,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:14,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:17,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:18,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:22,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:56:22,184 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:56:22,184 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:56:22,185 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:56:23,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:56:23,103 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:23,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:23,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:23,903 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:56:27,104 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:28,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:28,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:33,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:33,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:38,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:38,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:42,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:56:42,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:56:42,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:56:42,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:56:43,110 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:56:44,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:44,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:45,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:47,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:48,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:49,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:53,904 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:56:54,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:54,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:59,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:59,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:03,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:03,058 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:03,058 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:03,058 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:03,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:05,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:05,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:07,120 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:09,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:10,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:14,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:15,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:18,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:20,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:23,905 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:57:24,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:25,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:25,208 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:25,208 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:25,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:25,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:26,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:27,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:29,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:29,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:30,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:31,129 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:33,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:35,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:38,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:40,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:42,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:42,584 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:42,584 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:42,585 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:43,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:44,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:45,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:45,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:47,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:48,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:50,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:53,906 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:57:54,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:55,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:59,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:00,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:00,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:00,485 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:00,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:01,141 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:01,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:03,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:05,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:06,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:09,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:11,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:14,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:16,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:18,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:18,041 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:18,042 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:18,042 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:18,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:18,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:19,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:22,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:23,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:23,907 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:58:24,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:27,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:29,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:32,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:33,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:37,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:39,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:42,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:44,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:44,794 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:44,795 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:44,795 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:44,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:45,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:47,160 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:47,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:48,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:49,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:52,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:53,908 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:58:54,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:58,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:59,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:59,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:59,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:59,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:59,766 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:00,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:01,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:03,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:03,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:05,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:08,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:09,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:13,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:14,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:18,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:18,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:20,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:59:20,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:59:20,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:59:20,451 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:21,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:21,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:23,909 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:59:24,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:24,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:25,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:29,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:29,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:33,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:34,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:39,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:39,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:43,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:59:43,797 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:59:43,797 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:59:43,798 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:44,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:45,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:45,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:48,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:49,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:50,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:54,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:55,317 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:59:56,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:59,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:01,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:01,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:01,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:01,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:01,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:02,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:03,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:04,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:06,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:07,192 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:09,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:11,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:14,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:16,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:19,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:19,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:19,617 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:19,617 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:19,618 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:20,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:21,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:22,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:23,912 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:00:24,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:25,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:27,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:29,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:32,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:34,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:37,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:37,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:37,765 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:37,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:38,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:38,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:39,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:39,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:43,206 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:44,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:44,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:49,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:49,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:53,912 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:00:54,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:54,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:57,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:57,371 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:57,371 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:57,372 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:58,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:59,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:00,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:00,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:01,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:05,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:10,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:10,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:15,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:15,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:19,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:20,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:23,913 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:01:25,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:25,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:26,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:01:26,380 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:01:26,380 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:01:26,381 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:01:27,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:01:27,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:31,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:31,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:31,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:34,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:36,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:40,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:41,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:45,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:45,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:01:45,479 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:01:45,480 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:01:45,480 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:01:46,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:01:47,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:47,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:49,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:49,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:52,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:53,914 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:01:54,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:57,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:59,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:00,641 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:00,641 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:00,642 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:01,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:01,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:03,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:05,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:09,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:09,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:14,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:14,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:19,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:19,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:19,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:19,938 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:19,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:19,939 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:20,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:21,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:23,915 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:02:24,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:24,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:25,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:29,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:29,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:34,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:34,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:34,746 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:34,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:34,747 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:35,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:35,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:35,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:39,251 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:39,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:40,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:44,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:45,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:49,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:50,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:53,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:53,379 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:53,379 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:53,380 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:53,916 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:02:54,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:55,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:55,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:55,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:57,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:00,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:01,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:04,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:06,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:08,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:08,628 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:08,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:08,629 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:09,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:09,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:10,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:11,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:13,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:15,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:16,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:19,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:21,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:23,917 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:03:24,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:26,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:29,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:30,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:30,642 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:30,642 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:30,643 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:31,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:31,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:32,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:34,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:35,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:37,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:39,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:42,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:44,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:47,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:49,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:52,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:53,918 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:03:54,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:55,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:55,793 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:55,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:55,793 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:56,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:57,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:58,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:59,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:00,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:01,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:04,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:04,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:09,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:09,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:13,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:13,911 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:13,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:13,912 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:14,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:14,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:14,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:15,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:19,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:19,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:20,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:23,918 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:04:24,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:25,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:29,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:30,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:31,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:31,227 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:31,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:31,228 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:31,294 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:33,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:34,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:35,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:36,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:39,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:41,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:44,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:46,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:48,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:48,456 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:48,457 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:48,457 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:49,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:49,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:49,302 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:51,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:53,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:53,924 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:04:54,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:56,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:59,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:01,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:04,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:06,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:08,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:08,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:08,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:08,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:09,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:09,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:09,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:11,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:13,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:14,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:16,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:19,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:21,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:23,920 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:05:24,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:25,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:25,695 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:25,696 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:25,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:26,316 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:27,316 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:27,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:29,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:30,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:32,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:34,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:37,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:39,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:42,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:44,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:44,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:44,543 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:44,543 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:44,544 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:45,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:45,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:48,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:49,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:49,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:53,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:53,921 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:05:55,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:58,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:00,147 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:03,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:03,640 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:03,640 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:03,642 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:04,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:04,330 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:04,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:05,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:07,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:09,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:09,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:14,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:14,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:19,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:19,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:20,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:20,694 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:20,694 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:20,694 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:21,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:21,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:23,922 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:06:24,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:24,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:25,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:29,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:29,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:34,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:34,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:39,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:39,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:39,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:39,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:39,547 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:40,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:40,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:41,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:45,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:45,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:45,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:49,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:50,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:53,923 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:06:55,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:55,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:55,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:55,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:55,967 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:55,967 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:56,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:57,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:00,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:01,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:01,351 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:04,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:06,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:09,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:11,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:14,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:14,572 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:14,573 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:14,575 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:14,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:15,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:15,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:16,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:19,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:19,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:21,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:23,923 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:07:24,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:26,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:29,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:31,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:31,328 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:31,328 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:31,328 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:31,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:32,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:33,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:34,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:35,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:37,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:40,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:42,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:45,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:47,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:49,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:52,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:53,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:53,838 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:53,838 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:53,840 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:53,925 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:07:54,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:55,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:55,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:58,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:59,376 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:00,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:03,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:04,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:08,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:09,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:10,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:10,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:10,718 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:10,718 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:11,377 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:11,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:13,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:15,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:15,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:18,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:19,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:23,925 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:08:24,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:24,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:27,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:27,240 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:27,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:27,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:27,384 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:29,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:29,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:30,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:31,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:34,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:35,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:39,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:40,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:44,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:45,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:49,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:50,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:53,926 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:08:54,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:55,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:55,259 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:55,259 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:55,260 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:55,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:56,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:57,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:59,396 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:00,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:01,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:04,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:06,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:10,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:11,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:13,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:13,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:13,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:13,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:13,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:15,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:15,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:17,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:17,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:19,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:22,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:23,927 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:09:24,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:27,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:29,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:30,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:30,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:30,759 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:30,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:31,408 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:31,409 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:33,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:34,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:35,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:38,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:39,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:43,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:44,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:48,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:48,833 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:48,833 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:48,834 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:49,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:49,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:49,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:49,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:53,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:53,997 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:09:54,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:54,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:59,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:59,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:04,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:04,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:06,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:06,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:06,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:06,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:07,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:07,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:10,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:10,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:11,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:15,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:15,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:19,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:20,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:23,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:23,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:23,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:23,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:23,515 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:23,929 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:10:24,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:25,432 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:25,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:27,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:29,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:31,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:34,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:36,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:39,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:41,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:43,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:43,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:43,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:43,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:44,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:45,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:46,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:48,443 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:49,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:51,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:53,930 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:10:55,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:56,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:00,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:01,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:01,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:01,942 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:01,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:01,945 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:02,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:04,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:04,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:06,450 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:07,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:08,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:09,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:12,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:14,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:17,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:19,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:21,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:21,417 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:21,417 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:21,417 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:21,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:22,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:23,931 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:11:24,458 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:25,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:26,459 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:27,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:30,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:32,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:34,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:37,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:37,194 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:37,194 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:37,196 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:37,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:38,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:39,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:40,464 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:42,465 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:43,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:44,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:48,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:49,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:53,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:53,932 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:11:54,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:58,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:59,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:59,045 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:59,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:59,046 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:59,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:00,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:00,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:04,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:04,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:04,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:09,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:09,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:14,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:14,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:18,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:18,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:18,153 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:18,155 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:18,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:19,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:19,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:19,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:20,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:20,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:23,933 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:12:24,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:24,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:25,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:29,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:30,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:34,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:35,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:38,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:38,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:38,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:38,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:39,491 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:40,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:40,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:41,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:44,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:45,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:47,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:49,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:52,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:53,934 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:12:55,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:57,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:58,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:58,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:58,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:58,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:58,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:00,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:00,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:03,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:04,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:04,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:08,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:09,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:13,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:14,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:17,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:17,066 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:17,066 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:17,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:18,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:19,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:19,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:22,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:23,935 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:13:24,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:25,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:29,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:30,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:31,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:31,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:31,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:32,663 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:33,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:34,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:35,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:35,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:38,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:40,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:40,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:45,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:50,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:50,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:53,936 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:13:55,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:55,325 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:55,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:55,326 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:55,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:55,520 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:56,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:58,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:00,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:01,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:01,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:05,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:06,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:11,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:11,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:16,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:16,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:19,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:19,929 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:19,929 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:19,930 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:20,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:20,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:21,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:22,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:23,936 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:14:24,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:25,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:27,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:30,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:32,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:35,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:37,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:38,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:38,032 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:38,032 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:38,032 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:38,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:40,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:40,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:42,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:43,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:45,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:48,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:50,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:52,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:52,853 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:52,853 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:52,854 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:53,544 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:53,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:53,937 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:14:54,545 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:55,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:58,546 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:59,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:04,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:05,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:09,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:10,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:14,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:15,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:15,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:15,973 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:15,974 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:15,974 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:16,552 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:15:18,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:19,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:20,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:20,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:23,938 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:15:24,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:25,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:29,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:30,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:34,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:35,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:35,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:35,462 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:35,462 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:35,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:35,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:15:38,560 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:40,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:40,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:41,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:45,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:46,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:50,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:50,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:53,939 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:15:55,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:56,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:58,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:58,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:58,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:58,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:58,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:00,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:01,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:01,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:04,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:05,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:06,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:11,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:15,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:16,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:17,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:16:17,367 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:16:17,368 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:16:17,368 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:16:17,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:18,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:20,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:22,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:22,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:23,940 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:16:26,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:27,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:31,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:32,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:35,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:37,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:16:37,064 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:16:37,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:16:37,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:16:37,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:38,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:38,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:40,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:42,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:43,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:45,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:48,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:50,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:53,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:53,941 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:16:55,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:58,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:00,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:02,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:02,237 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:02,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:02,238 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:02,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:04,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:04,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:05,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:06,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:09,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:10,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:14,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:15,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:19,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:20,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:20,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:20,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:20,344 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:20,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:20,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:22,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:23,942 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:17:24,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:24,943 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:26,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:29,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:31,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:34,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:35,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:39,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:41,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:42,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:42,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:42,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:42,218 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:42,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:44,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:45,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:46,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:46,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:50,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:50,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:53,943 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:17:55,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:55,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:59,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:59,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:59,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:59,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:00,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:01,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:01,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:02,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:04,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:05,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:07,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:10,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:12,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:15,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:15,411 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:15,411 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:15,411 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:15,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:15,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:16,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:17,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:20,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:20,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:22,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:23,943 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:18:26,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:27,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:30,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:30,306 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:30,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:30,307 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:30,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:31,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:32,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:33,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:34,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:35,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:38,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:40,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:43,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:45,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:46,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:46,850 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:46,850 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:46,850 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:47,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:48,633 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:48,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:50,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:52,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:53,944 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:18:53,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:55,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:58,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:00,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:01,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:01,565 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:01,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:01,566 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:01,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:02,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:04,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:05,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:06,642 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:09,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:10,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:14,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:15,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:16,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:16,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:16,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:16,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:16,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:18,646 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:20,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:20,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:20,647 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:23,945 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:19:25,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:25,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:30,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:30,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:35,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:35,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:39,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:39,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:39,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:39,243 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:39,653 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:40,654 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:40,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:41,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:44,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:46,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:46,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:50,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:51,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:53,946 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:19:56,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:56,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:01,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:01,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:01,784 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:01,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:01,785 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:02,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:02,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:04,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:05,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:06,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:07,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:10,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:12,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:15,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:17,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:20,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:22,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:23,947 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:20:25,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:27,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:27,537 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:27,537 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:27,538 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:27,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:28,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:28,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:31,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:32,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:33,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:35,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:38,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:40,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:43,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:45,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:48,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:48,372 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:48,373 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:48,373 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:48,678 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:49,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:50,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:50,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:50,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:52,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:53,948 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:20:54,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:56,103 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:59,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:01,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:04,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:04,504 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:04,504 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:04,504 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:04,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:05,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:06,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:06,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:08,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:10,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:11,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:15,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:16,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:20,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:21,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:21,030 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:21,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:21,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:21,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:21,690 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:22,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:23,949 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:21:26,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:26,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:26,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:31,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:31,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:36,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:36,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:41,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:41,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:46,154 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:46,155 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:46,155 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:46,156 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:46,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:46,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:47,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:48,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:50,702 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:51,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:52,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:53,950 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:21:56,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:57,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:01,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:02,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:06,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:06,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:06,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:06,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:06,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:08,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:08,709 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:10,710 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:12,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:13,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:17,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:18,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:21,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:23,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:23,951 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:22:25,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:25,756 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:25,757 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:25,757 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:26,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:26,716 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:27,487 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:28,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:30,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:32,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:33,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:36,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:38,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:41,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:43,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:43,849 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:43,849 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:43,850 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:44,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:44,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:44,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:46,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:48,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:50,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:51,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:53,952 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:22:55,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:56,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:00,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:01,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:03,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:03,432 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:03,432 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:03,432 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:03,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:04,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:06,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:06,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:08,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:12,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:12,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:17,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:17,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:21,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:22,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:23,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:23,280 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:23,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:23,281 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:23,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:23,954 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:23:24,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:27,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:27,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:28,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:32,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:32,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:36,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:37,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:42,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:42,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:43,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:43,048 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:43,048 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:43,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:43,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:44,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:47,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:48,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:48,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:51,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:53,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:53,955 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:23:56,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:58,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:00,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:00,880 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:00,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:00,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:01,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:02,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:02,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:04,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:04,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:06,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:09,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:11,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:14,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:16,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:17,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:17,780 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:17,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:17,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:18,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:18,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:19,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:21,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:22,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:23,956 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:24:24,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:26,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:29,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:31,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:34,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:36,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:36,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:36,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:36,343 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:36,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:36,767 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:38,768 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:40,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:40,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:42,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:45,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:47,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:50,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:51,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:53,957 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:24:55,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:57,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:00,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:04,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:04,362 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:04,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:04,364 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:04,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:06,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:06,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:07,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:08,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:11,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:12,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:16,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:17,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:19,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:19,651 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:19,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:19,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:20,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:20,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:21,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:22,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:23,958 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:25:24,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:27,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:27,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:32,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:32,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:37,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:37,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:41,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:41,235 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:41,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:41,237 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:41,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:42,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:42,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:43,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:44,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:46,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:47,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:48,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:52,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:53,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:53,959 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:25:58,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:58,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:58,147 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:58,147 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:58,148 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:58,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:58,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:59,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:02,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:03,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:04,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:07,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:09,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:12,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:14,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:15,316 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:15,318 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:15,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:16,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:18,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:18,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:19,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:20,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:23,959 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:26:24,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:28,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:29,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:33,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:34,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:37,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:37,223 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:37,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:37,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:37,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:37,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:38,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:40,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:40,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:42,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:42,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:45,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:47,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:50,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:52,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:52,198 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:52,198 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:52,200 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:52,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:52,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:52,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:53,961 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:26:55,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:56,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:57,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:00,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:02,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:05,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:07,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:10,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:12,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:13,416 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:13,417 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:13,417 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:13,418 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:13,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:14,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:18,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:18,833 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:21,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:23,961 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:27:26,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:28,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:31,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:31,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:31,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:31,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:31,837 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:32,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:32,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:33,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:34,732 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 09:27:34,733 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:34,736 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,736 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,738 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,739 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:34,739 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,739 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:36,839 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:37,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:39,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:39,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:39,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:42,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:47,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:50,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:52,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:53,962 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:27:54,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:57,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:59,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:28:02,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:28:03,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 09:28:04,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive diff --git a/wandb/debug.log b/wandb/debug.log index 9140334e1d822c725b52935727a1b50d31f10587..091a026ceaa4b66c1f3159f811bd90b79f298893 100644 --- a/wandb/debug.log +++ b/wandb/debug.log @@ -1,32 +1,31 @@ -2024-04-11 03:59:09,934 INFO MainThread:145 [wandb_setup.py:_flush():76] Current SDK version is 0.16.5 -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Configure stats pid to 145 -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Loading settings from /kaggle/working/wandb/settings -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': ''} -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Applying login settings: {} -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_init.py:_log_setup():527] Logging user logs to /kaggle/working/wandb/run-20240411_035909-21esl9hl/logs/debug.log -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_init.py:_log_setup():528] Logging internal logs to /kaggle/working/wandb/run-20240411_035909-21esl9hl/logs/debug-internal.log -2024-04-11 03:59:09,935 INFO MainThread:145 [wandb_init.py:_jupyter_setup():473] configuring jupyter hooks -2024-04-11 03:59:09,936 INFO MainThread:145 [wandb_init.py:init():567] calling init triggers -2024-04-11 03:59:09,936 INFO MainThread:145 [wandb_init.py:init():574] wandb.init called with sweep_config: {} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Current SDK version is 0.16.5 +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Configure stats pid to 1050 +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Loading settings from /kaggle/working/wandb/settings +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': ''} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_init.py:_log_setup():527] Logging user logs to /kaggle/working/wandb/run-20240411_040722-rimyh9a1/logs/debug.log +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_init.py:_log_setup():528] Logging internal logs to /kaggle/working/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_init.py:_jupyter_setup():473] configuring jupyter hooks +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():567] calling init triggers +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():574] wandb.init called with sweep_config: {} config: {} -2024-04-11 03:59:09,936 INFO MainThread:145 [wandb_init.py:init():617] starting backend -2024-04-11 03:59:09,936 INFO MainThread:145 [wandb_init.py:init():621] setting up manager -2024-04-11 03:59:09,938 INFO MainThread:145 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2024-04-11 03:59:09,940 INFO MainThread:145 [wandb_init.py:init():629] backend started and connected -2024-04-11 03:59:09,953 INFO MainThread:145 [wandb_run.py:_label_probe_notebook():1299] probe notebook -2024-04-11 03:59:10,362 INFO MainThread:145 [wandb_init.py:init():721] updated telemetry -2024-04-11 03:59:10,366 INFO MainThread:145 [wandb_init.py:init():754] communicating run to backend with 90.0 second timeout -2024-04-11 03:59:10,526 INFO MainThread:145 [wandb_run.py:_on_init():2344] communicating current version -2024-04-11 03:59:10,613 INFO MainThread:145 [wandb_run.py:_on_init():2353] got version response upgrade_message: "wandb version 0.16.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():617] starting backend +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():621] setting up manager +2024-04-11 04:07:22,847 INFO MainThread:1050 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-04-11 04:07:22,848 INFO MainThread:1050 [wandb_init.py:init():629] backend started and connected +2024-04-11 04:07:22,865 INFO MainThread:1050 [wandb_run.py:_label_probe_notebook():1299] probe notebook +2024-04-11 04:07:23,106 INFO MainThread:1050 [wandb_init.py:init():721] updated telemetry +2024-04-11 04:07:23,108 INFO MainThread:1050 [wandb_init.py:init():754] communicating run to backend with 90.0 second timeout +2024-04-11 04:07:23,235 INFO MainThread:1050 [wandb_run.py:_on_init():2344] communicating current version +2024-04-11 04:07:23,323 INFO MainThread:1050 [wandb_run.py:_on_init():2353] got version response upgrade_message: "wandb version 0.16.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" -2024-04-11 03:59:10,614 INFO MainThread:145 [wandb_init.py:init():805] starting run threads in backend -2024-04-11 03:59:26,715 INFO MainThread:145 [wandb_run.py:_console_start():2323] atexit reg -2024-04-11 03:59:26,715 INFO MainThread:145 [wandb_run.py:_redirect():2178] redirect: wrap_raw -2024-04-11 03:59:26,716 INFO MainThread:145 [wandb_run.py:_redirect():2243] Wrapping output streams. -2024-04-11 03:59:26,716 INFO MainThread:145 [wandb_run.py:_redirect():2268] Redirects installed. -2024-04-11 03:59:26,717 INFO MainThread:145 [wandb_init.py:init():848] run started, returning control to user process -2024-04-11 03:59:26,723 INFO MainThread:145 [wandb_run.py:_config_callback():1347] config_cb None None {'vocab_size': 250880, 'hidden_size': 1024, 'n_layer': 24, 'n_head': 16, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'use_cache': False, 'pretraining_tp': 1, 'apply_residual_connection_post_layernorm': False, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'bos_token_id': 1, 'eos_token_id': 2, 'slow_but_exact': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['BloomForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': 3, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'bigscience/bloomz-560m', 'transformers_version': '4.39.3', 'attention_softmax_in_fp32': True, 'bias_dropout_fusion': True, 'unk_token_id': 0, 'masked_softmax_fusion': True, 'model_type': 'bloom', 'n_inner': None, 'offset_alibi': 100, 'seq_length': 2048, 'skip_bias_add': True, 'skip_bias_add_qkv': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/kaggle/working/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 3e-05, 'weight_decay': 0.0001, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 5, 'max_steps': 10, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/kaggle/working/runs/Apr11_03-48-59_321ddc84c049', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 20, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 20, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/kaggle/working/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None} +2024-04-11 04:07:23,324 INFO MainThread:1050 [wandb_init.py:init():805] starting run threads in backend +2024-04-11 04:07:39,317 INFO MainThread:1050 [wandb_run.py:_console_start():2323] atexit reg +2024-04-11 04:07:39,317 INFO MainThread:1050 [wandb_run.py:_redirect():2178] redirect: wrap_raw +2024-04-11 04:07:39,318 INFO MainThread:1050 [wandb_run.py:_redirect():2243] Wrapping output streams. +2024-04-11 04:07:39,318 INFO MainThread:1050 [wandb_run.py:_redirect():2268] Redirects installed. +2024-04-11 04:07:39,319 INFO MainThread:1050 [wandb_init.py:init():848] run started, returning control to user process +2024-04-11 04:07:39,324 INFO MainThread:1050 [wandb_run.py:_config_callback():1347] config_cb None None {'vocab_size': 250880, 'hidden_size': 1024, 'n_layer': 24, 'n_head': 16, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'use_cache': False, 'pretraining_tp': 1, 'apply_residual_connection_post_layernorm': False, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'bos_token_id': 1, 'eos_token_id': 2, 'slow_but_exact': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['BloomForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': 3, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'bigscience/bloomz-560m', 'transformers_version': '4.39.3', 'attention_softmax_in_fp32': True, 'bias_dropout_fusion': True, 'unk_token_id': 0, 'masked_softmax_fusion': True, 'model_type': 'bloom', 'n_inner': None, 'offset_alibi': 100, 'seq_length': 2048, 'skip_bias_add': True, 'skip_bias_add_qkv': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/kaggle/working/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 3e-05, 'weight_decay': 0.001, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 5, 'max_steps': 20000, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/kaggle/working/runs/Apr11_04-07-17_321ddc84c049', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 20, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 20, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/kaggle/working/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None} diff --git a/wandb/run-20240411_035909-21esl9hl/files/output.log b/wandb/run-20240411_035909-21esl9hl/files/output.log index 2612c6d6d46b22db7c8149845b693fdd7ed3d8b3..9054930049617ea74f51342c76598911b67e203a 100644 --- a/wandb/run-20240411_035909-21esl9hl/files/output.log +++ b/wandb/run-20240411_035909-21esl9hl/files/output.log @@ -101,3 +101,4 @@ To disable this warning, you can either: - Avoid using `tokenizers` before the fork if possible - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) /opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") \ No newline at end of file diff --git a/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json b/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json index bb261ab82e72d6e1654e37a5275a3bbfe68f2322..cb7e75dc869cbbf7b6410e98b75af33f4b321973 100644 --- a/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json +++ b/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json @@ -1 +1 @@ -{"train_runtime": 14.0567, "train_samples_per_second": 2.846, "train_steps_per_second": 0.711, "total_flos": 28907911741440.0, "train_loss": 21.220675659179687, "train/epoch": 0.0, "train/global_step": 10, "_timestamp": 1712808008.9437401, "_runtime": 59.003005027770996, "_step": 0} \ No newline at end of file +{"train_runtime": 14.0567, "train_samples_per_second": 2.846, "train_steps_per_second": 0.711, "total_flos": 28907911741440.0, "train_loss": 21.220675659179687, "train/epoch": 0.0, "train/global_step": 10, "_timestamp": 1712808008.9437401, "_runtime": 59.003005027770996, "_step": 0, "_wandb": {"runtime": 91}} \ No newline at end of file diff --git a/wandb/run-20240411_035909-21esl9hl/logs/debug-internal.log b/wandb/run-20240411_035909-21esl9hl/logs/debug-internal.log index 6d27e973415956c682f3d3f4931936c37532a9bc..2af29c1edb12f6e281d26fbf93b20d2583d3e930 100644 --- a/wandb/run-20240411_035909-21esl9hl/logs/debug-internal.log +++ b/wandb/run-20240411_035909-21esl9hl/logs/debug-internal.log @@ -116,3 +116,161 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 2024-04-11 04:00:27,786 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report 2024-04-11 04:00:32,787 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report 2024-04-11 04:00:37,788 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:00:40,648 DEBUG SenderThread:191 [sender.py:send():379] send: stats +2024-04-11 04:00:42,143 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:00:42,144 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:00:42,184 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:00:42,200 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: pause +2024-04-11 04:00:42,200 INFO HandlerThread:191 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-11 04:00:42,200 INFO HandlerThread:191 [system_monitor.py:finish():203] Stopping system monitor +2024-04-11 04:00:42,200 DEBUG SystemMonitor:191 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-11 04:00:42,200 DEBUG SystemMonitor:191 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-11 04:00:42,202 INFO HandlerThread:191 [interfaces.py:finish():202] Joined cpu monitor +2024-04-11 04:00:42,202 INFO HandlerThread:191 [interfaces.py:finish():202] Joined disk monitor +2024-04-11 04:00:42,211 INFO HandlerThread:191 [interfaces.py:finish():202] Joined gpu monitor +2024-04-11 04:00:42,212 INFO HandlerThread:191 [interfaces.py:finish():202] Joined memory monitor +2024-04-11 04:00:42,212 INFO HandlerThread:191 [interfaces.py:finish():202] Joined network monitor +2024-04-11 04:00:42,234 DEBUG SenderThread:191 [sender.py:send():379] send: stats +2024-04-11 04:00:43,235 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:00:48,235 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:00:53,236 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:00:57,141 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:00:57,142 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:00:57,182 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:00:58,263 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:01:03,264 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:01:08,265 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:01:12,141 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:01:12,142 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:01:12,182 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:01:13,266 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:01:18,267 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:01:23,268 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:01:23,392 DEBUG SenderThread:191 [sender.py:send():379] send: exit +2024-04-11 04:01:23,392 INFO SenderThread:191 [sender.py:send_exit():586] handling exit code: 0 +2024-04-11 04:01:23,392 INFO SenderThread:191 [sender.py:send_exit():588] handling runtime: 91 +2024-04-11 04:01:23,394 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:01:23,394 INFO SenderThread:191 [sender.py:send_exit():594] send defer +2024-04-11 04:01:23,394 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,394 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 0 +2024-04-11 04:01:23,395 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,395 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-04-11 04:01:23,395 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 1 +2024-04-11 04:01:23,395 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,395 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 1 +2024-04-11 04:01:23,395 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,395 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-04-11 04:01:23,395 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 2 +2024-04-11 04:01:23,395 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,396 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 2 +2024-04-11 04:01:23,396 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,396 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-04-11 04:01:23,396 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 3 +2024-04-11 04:01:23,396 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,396 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 3 +2024-04-11 04:01:23,396 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,396 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-04-11 04:01:23,396 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 4 +2024-04-11 04:01:23,396 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,397 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 4 +2024-04-11 04:01:23,397 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,397 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-04-11 04:01:23,397 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 5 +2024-04-11 04:01:23,397 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,397 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 5 +2024-04-11 04:01:23,397 DEBUG SenderThread:191 [sender.py:send():379] send: summary +2024-04-11 04:01:23,398 INFO SenderThread:191 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:01:23,398 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,398 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-04-11 04:01:23,398 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 6 +2024-04-11 04:01:23,398 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,398 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 6 +2024-04-11 04:01:23,398 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,399 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-04-11 04:01:23,399 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 7 +2024-04-11 04:01:23,399 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:01:23,399 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:23,399 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 7 +2024-04-11 04:01:23,399 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:23,399 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-04-11 04:01:23,565 INFO Thread-12 :191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json +2024-04-11 04:01:24,392 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:01:24,961 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 8 +2024-04-11 04:01:24,961 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:01:24,961 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:24,961 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 8 +2024-04-11 04:01:24,962 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:24,962 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-04-11 04:01:24,962 INFO SenderThread:191 [job_builder.py:build():318] Attempting to build job artifact +2024-04-11 04:01:24,964 INFO SenderThread:191 [job_builder.py:_get_source_type():466] no source found +2024-04-11 04:01:24,964 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 9 +2024-04-11 04:01:24,964 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:24,964 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 9 +2024-04-11 04:01:24,964 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:24,964 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-04-11 04:01:24,964 INFO SenderThread:191 [dir_watcher.py:finish():358] shutting down directory watcher +2024-04-11 04:01:25,393 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:01:25,566 INFO SenderThread:191 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log +2024-04-11 04:01:25,567 INFO SenderThread:191 [dir_watcher.py:finish():388] scan: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files +2024-04-11 04:01:25,567 INFO SenderThread:191 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/config.yaml config.yaml +2024-04-11 04:01:25,567 INFO SenderThread:191 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/requirements.txt requirements.txt +2024-04-11 04:01:25,568 INFO SenderThread:191 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/conda-environment.yaml conda-environment.yaml +2024-04-11 04:01:25,574 INFO SenderThread:191 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json wandb-summary.json +2024-04-11 04:01:25,574 INFO SenderThread:191 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/wandb-metadata.json wandb-metadata.json +2024-04-11 04:01:25,574 INFO SenderThread:191 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log output.log +2024-04-11 04:01:25,575 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 10 +2024-04-11 04:01:25,575 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:01:25,578 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:25,578 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 10 +2024-04-11 04:01:25,579 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:25,579 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-04-11 04:01:25,579 INFO SenderThread:191 [file_pusher.py:finish():172] shutting down file pusher +2024-04-11 04:01:25,725 INFO wandb-upload_1:191 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/requirements.txt +2024-04-11 04:01:25,772 INFO wandb-upload_0:191 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/config.yaml +2024-04-11 04:01:25,777 INFO wandb-upload_3:191 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/output.log +2024-04-11 04:01:25,846 INFO wandb-upload_2:191 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_035909-21esl9hl/files/wandb-summary.json +2024-04-11 04:01:26,046 INFO Thread-11 (_thread_body):191 [sender.py:transition_state():614] send defer: 11 +2024-04-11 04:01:26,046 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:26,047 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 11 +2024-04-11 04:01:26,047 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:26,047 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-04-11 04:01:26,047 INFO SenderThread:191 [file_pusher.py:join():178] waiting for file pusher +2024-04-11 04:01:26,047 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 12 +2024-04-11 04:01:26,048 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:26,048 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 12 +2024-04-11 04:01:26,048 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:26,048 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-04-11 04:01:26,048 INFO SenderThread:191 [file_stream.py:finish():614] file stream finish called +2024-04-11 04:01:26,118 INFO SenderThread:191 [file_stream.py:finish():618] file stream finish is done +2024-04-11 04:01:26,118 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 13 +2024-04-11 04:01:26,118 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:26,118 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 13 +2024-04-11 04:01:26,119 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:26,119 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-04-11 04:01:26,119 INFO SenderThread:191 [sender.py:transition_state():614] send defer: 14 +2024-04-11 04:01:26,119 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:01:26,119 INFO HandlerThread:191 [handler.py:handle_request_defer():172] handle defer: 14 +2024-04-11 04:01:26,120 DEBUG SenderThread:191 [sender.py:send():379] send: final +2024-04-11 04:01:26,120 DEBUG SenderThread:191 [sender.py:send():379] send: footer +2024-04-11 04:01:26,120 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: defer +2024-04-11 04:01:26,120 INFO SenderThread:191 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-04-11 04:01:26,121 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:01:26,121 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:01:26,122 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:01:26,122 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:01:26,123 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: server_info +2024-04-11 04:01:26,123 DEBUG SenderThread:191 [sender.py:send_request():406] send_request: server_info +2024-04-11 04:01:26,127 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: get_summary +2024-04-11 04:01:26,127 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: sampled_history +2024-04-11 04:01:26,128 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:01:26,139 INFO MainThread:191 [wandb_run.py:_footer_history_summary_info():3920] rendering history +2024-04-11 04:01:26,139 INFO MainThread:191 [wandb_run.py:_footer_history_summary_info():3952] rendering summary +2024-04-11 04:01:26,139 INFO MainThread:191 [wandb_run.py:_footer_sync_info():3879] logging synced files +2024-04-11 04:01:26,140 DEBUG HandlerThread:191 [handler.py:handle_request():146] handle_request: shutdown +2024-04-11 04:01:26,140 INFO HandlerThread:191 [handler.py:finish():866] shutting down handler +2024-04-11 04:01:27,084 WARNING StreamThr :191 [internal.py:is_dead():414] Internal process exiting, parent pid 145 disappeared +2024-04-11 04:01:27,084 ERROR StreamThr :191 [internal.py:wandb_internal():152] Internal process shutdown. +2024-04-11 04:01:27,123 INFO WriterThread:191 [datastore.py:close():296] close: /kaggle/working/wandb/run-20240411_035909-21esl9hl/run-21esl9hl.wandb +2024-04-11 04:01:27,139 INFO SenderThread:191 [sender.py:finish():1546] shutting down sender +2024-04-11 04:01:27,139 INFO SenderThread:191 [file_pusher.py:finish():172] shutting down file pusher +2024-04-11 04:01:27,139 INFO SenderThread:191 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240411_035909-21esl9hl/logs/debug.log b/wandb/run-20240411_035909-21esl9hl/logs/debug.log index 9140334e1d822c725b52935727a1b50d31f10587..78de1a0c7c198b5acd7583d82474743bb7978d3f 100644 --- a/wandb/run-20240411_035909-21esl9hl/logs/debug.log +++ b/wandb/run-20240411_035909-21esl9hl/logs/debug.log @@ -30,3 +30,5 @@ config: {} 2024-04-11 03:59:26,716 INFO MainThread:145 [wandb_run.py:_redirect():2268] Redirects installed. 2024-04-11 03:59:26,717 INFO MainThread:145 [wandb_init.py:init():848] run started, returning control to user process 2024-04-11 03:59:26,723 INFO MainThread:145 [wandb_run.py:_config_callback():1347] config_cb None None {'vocab_size': 250880, 'hidden_size': 1024, 'n_layer': 24, 'n_head': 16, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'use_cache': False, 'pretraining_tp': 1, 'apply_residual_connection_post_layernorm': False, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'bos_token_id': 1, 'eos_token_id': 2, 'slow_but_exact': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['BloomForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': 3, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'bigscience/bloomz-560m', 'transformers_version': '4.39.3', 'attention_softmax_in_fp32': True, 'bias_dropout_fusion': True, 'unk_token_id': 0, 'masked_softmax_fusion': True, 'model_type': 'bloom', 'n_inner': None, 'offset_alibi': 100, 'seq_length': 2048, 'skip_bias_add': True, 'skip_bias_add_qkv': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/kaggle/working/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 3e-05, 'weight_decay': 0.0001, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 5, 'max_steps': 10, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/kaggle/working/runs/Apr11_03-48-59_321ddc84c049', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 20, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 20, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/kaggle/working/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None} +2024-04-11 04:00:42,199 INFO MainThread:145 [jupyter.py:save_ipynb():373] not saving jupyter notebook +2024-04-11 04:00:42,200 INFO MainThread:145 [wandb_init.py:_pause_backend():438] pausing backend diff --git a/wandb/run-20240411_035909-21esl9hl/run-21esl9hl.wandb b/wandb/run-20240411_035909-21esl9hl/run-21esl9hl.wandb index d6ddc14e2217dd1df9d212efff24563936d988aa..53fcf71dd9a5b579b738cb396f56a5018594f819 100644 Binary files a/wandb/run-20240411_035909-21esl9hl/run-21esl9hl.wandb and b/wandb/run-20240411_035909-21esl9hl/run-21esl9hl.wandb differ diff --git a/wandb/run-20240411_040208-txpa3x4k/files/conda-environment.yaml b/wandb/run-20240411_040208-txpa3x4k/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/run-20240411_040208-txpa3x4k/files/config.yaml b/wandb/run-20240411_040208-txpa3x4k/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53b02e4b19680a1747c6f6eaa4ada34a08b68e96 --- /dev/null +++ b/wandb/run-20240411_040208-txpa3x4k/files/config.yaml @@ -0,0 +1,701 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.10.13 + cli_version: 0.16.5 + framework: huggingface + huggingface_version: 4.39.3 + is_jupyter_run: true + is_kaggle_kernel: true + start_time: 1712808128.0 + t: + 1: + - 1 + - 2 + - 3 + - 5 + - 11 + - 12 + - 49 + - 51 + - 53 + - 55 + - 71 + - 84 + - 98 + - 105 + 2: + - 1 + - 2 + - 3 + - 5 + - 11 + - 12 + - 49 + - 51 + - 53 + - 55 + - 71 + - 84 + - 98 + - 105 + 3: + - 7 + - 23 + 4: 3.10.13 + 5: 0.16.5 + 6: 4.39.3 + 8: + - 1 + - 2 + - 5 + 9: + 1: transformers_trainer + 13: linux-x86_64 + m: + - 1: train/global_step + 6: + - 3 + - 1: train/loss + 5: 1 + 6: + - 1 + - 1: train/grad_norm + 5: 1 + 6: + - 1 + - 1: train/learning_rate + 5: 1 + 6: + - 1 + - 1: train/epoch + 5: 1 + 6: + - 1 +vocab_size: + desc: null + value: 250880 +hidden_size: + desc: null + value: 1024 +n_layer: + desc: null + value: 24 +n_head: + desc: null + value: 16 +layer_norm_epsilon: + desc: null + value: 1.0e-05 +initializer_range: + desc: null + value: 0.02 +use_cache: + desc: null + value: false +pretraining_tp: + desc: null + value: 1 +apply_residual_connection_post_layernorm: + desc: null + value: false +hidden_dropout: + desc: null + value: 0.0 +attention_dropout: + desc: null + value: 0.0 +bos_token_id: + desc: null + value: 1 +eos_token_id: + desc: null + value: 2 +slow_but_exact: + desc: null + value: false +return_dict: + desc: null + value: true +output_hidden_states: + desc: null + value: false +output_attentions: + desc: null + value: false +torchscript: + desc: null + value: false +torch_dtype: + desc: null + value: null +use_bfloat16: + desc: null + value: false +tf_legacy_loss: + desc: null + value: false +pruned_heads: + desc: null + value: {} +tie_word_embeddings: + desc: null + value: true +chunk_size_feed_forward: + desc: null + value: 0 +is_encoder_decoder: + desc: null + value: false +is_decoder: + desc: null + value: false +cross_attention_hidden_size: + desc: null + value: null +add_cross_attention: + desc: null + value: false +tie_encoder_decoder: + desc: null + value: false +max_length: + desc: null + value: 20 +min_length: + desc: null + value: 0 +do_sample: + desc: null + value: false +early_stopping: + desc: null + value: false +num_beams: + desc: null + value: 1 +num_beam_groups: + desc: null + value: 1 +diversity_penalty: + desc: null + value: 0.0 +temperature: + desc: null + value: 1.0 +top_k: + desc: null + value: 50 +top_p: + desc: null + value: 1.0 +typical_p: + desc: null + value: 1.0 +repetition_penalty: + desc: null + value: 1.0 +length_penalty: + desc: null + value: 1.0 +no_repeat_ngram_size: + desc: null + value: 0 +encoder_no_repeat_ngram_size: + desc: null + value: 0 +bad_words_ids: + desc: null + value: null +num_return_sequences: + desc: null + value: 1 +output_scores: + desc: null + value: false +return_dict_in_generate: + desc: null + value: false +forced_bos_token_id: + desc: null + value: null +forced_eos_token_id: + desc: null + value: null +remove_invalid_values: + desc: null + value: false +exponential_decay_length_penalty: + desc: null + value: null +suppress_tokens: + desc: null + value: null +begin_suppress_tokens: + desc: null + value: null +architectures: + desc: null + value: + - BloomForCausalLM +finetuning_task: + desc: null + value: null +id2label: + desc: null + value: + '0': LABEL_0 + '1': LABEL_1 +label2id: + desc: null + value: + LABEL_0: 0 + LABEL_1: 1 +tokenizer_class: + desc: null + value: null +prefix: + desc: null + value: null +pad_token_id: + desc: null + value: 3 +sep_token_id: + desc: null + value: null +decoder_start_token_id: + desc: null + value: null +task_specific_params: + desc: null + value: null +problem_type: + desc: null + value: null +_name_or_path: + desc: null + value: bigscience/bloomz-560m +transformers_version: + desc: null + value: 4.39.3 +attention_softmax_in_fp32: + desc: null + value: true +bias_dropout_fusion: + desc: null + value: true +unk_token_id: + desc: null + value: 0 +masked_softmax_fusion: + desc: null + value: true +model_type: + desc: null + value: bloom +n_inner: + desc: null + value: null +offset_alibi: + desc: null + value: 100 +seq_length: + desc: null + value: 2048 +skip_bias_add: + desc: null + value: true +skip_bias_add_qkv: + desc: null + value: false +quantization_config: + desc: null + value: + quant_method: QuantizationMethod.BITS_AND_BYTES + _load_in_8bit: false + _load_in_4bit: true + llm_int8_threshold: 6.0 + llm_int8_skip_modules: null + llm_int8_enable_fp32_cpu_offload: false + llm_int8_has_fp16_weight: false + bnb_4bit_quant_type: nf4 + bnb_4bit_use_double_quant: false + bnb_4bit_compute_dtype: float16 + bnb_4bit_quant_storage: uint8 + load_in_4bit: true + load_in_8bit: false +output_dir: + desc: null + value: /kaggle/working/ +overwrite_output_dir: + desc: null + value: false +do_train: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +evaluation_strategy: + desc: null + value: 'no' +prediction_loss_only: + desc: null + value: false +per_device_train_batch_size: + desc: null + value: 8 +per_device_eval_batch_size: + desc: null + value: 8 +per_gpu_train_batch_size: + desc: null + value: null +per_gpu_eval_batch_size: + desc: null + value: null +gradient_accumulation_steps: + desc: null + value: 1 +eval_accumulation_steps: + desc: null + value: null +eval_delay: + desc: null + value: 0 +learning_rate: + desc: null + value: 3.0e-05 +weight_decay: + desc: null + value: 0.001 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.999 +adam_epsilon: + desc: null + value: 1.0e-08 +max_grad_norm: + desc: null + value: 0.3 +num_train_epochs: + desc: null + value: 5 +max_steps: + desc: null + value: 7000 +lr_scheduler_type: + desc: null + value: cosine +lr_scheduler_kwargs: + desc: null + value: {} +warmup_ratio: + desc: null + value: 0.03 +warmup_steps: + desc: null + value: 0 +log_level: + desc: null + value: passive +log_level_replica: + desc: null + value: warning +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: /kaggle/working/runs/Apr11_04-02-02_321ddc84c049 +logging_strategy: + desc: null + value: steps +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 20 +logging_nan_inf_filter: + desc: null + value: true +save_strategy: + desc: null + value: steps +save_steps: + desc: null + value: 20 +save_total_limit: + desc: null + value: 1 +save_safetensors: + desc: null + value: true +save_on_each_node: + desc: null + value: false +save_only_model: + desc: null + value: false +no_cuda: + desc: null + value: false +use_cpu: + desc: null + value: false +use_mps_device: + desc: null + value: false +seed: + desc: null + value: 42 +data_seed: + desc: null + value: null +jit_mode_eval: + desc: null + value: false +use_ipex: + desc: null + value: false +bf16: + desc: null + value: false +fp16: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +half_precision_backend: + desc: null + value: auto +bf16_full_eval: + desc: null + value: false +fp16_full_eval: + desc: null + value: false +tf32: + desc: null + value: null +local_rank: + desc: null + value: 0 +ddp_backend: + desc: null + value: null +tpu_num_cores: + desc: null + value: null +tpu_metrics_debug: + desc: null + value: false +debug: + desc: null + value: [] +dataloader_drop_last: + desc: null + value: false +eval_steps: + desc: null + value: null +dataloader_num_workers: + desc: null + value: 8 +dataloader_prefetch_factor: + desc: null + value: null +past_index: + desc: null + value: -1 +run_name: + desc: null + value: /kaggle/working/ +disable_tqdm: + desc: null + value: false +remove_unused_columns: + desc: null + value: true +label_names: + desc: null + value: null +load_best_model_at_end: + desc: null + value: false +metric_for_best_model: + desc: null + value: null +greater_is_better: + desc: null + value: null +ignore_data_skip: + desc: null + value: false +fsdp: + desc: null + value: [] +fsdp_min_num_params: + desc: null + value: 0 +fsdp_config: + desc: null + value: + min_num_params: 0 + xla: false + xla_fsdp_v2: false + xla_fsdp_grad_ckpt: false +fsdp_transformer_layer_cls_to_wrap: + desc: null + value: null +accelerator_config: + desc: null + value: + split_batches: false + dispatch_batches: null + even_batches: true + use_seedable_sampler: true +deepspeed: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +optim: + desc: null + value: paged_adamw_32bit +optim_args: + desc: null + value: null +adafactor: + desc: null + value: false +group_by_length: + desc: null + value: false +length_column_name: + desc: null + value: length +report_to: + desc: null + value: + - tensorboard + - wandb +ddp_find_unused_parameters: + desc: null + value: null +ddp_bucket_cap_mb: + desc: null + value: null +ddp_broadcast_buffers: + desc: null + value: null +dataloader_pin_memory: + desc: null + value: true +dataloader_persistent_workers: + desc: null + value: false +skip_memory_metrics: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +push_to_hub: + desc: null + value: false +resume_from_checkpoint: + desc: null + value: null +hub_model_id: + desc: null + value: null +hub_strategy: + desc: null + value: every_save +hub_token: + desc: null + value: +hub_private_repo: + desc: null + value: false +hub_always_push: + desc: null + value: false +gradient_checkpointing: + desc: null + value: true +gradient_checkpointing_kwargs: + desc: null + value: null +include_inputs_for_metrics: + desc: null + value: false +fp16_backend: + desc: null + value: auto +push_to_hub_model_id: + desc: null + value: null +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: +mp_parameters: + desc: null + value: '' +auto_find_batch_size: + desc: null + value: true +full_determinism: + desc: null + value: false +torchdynamo: + desc: null + value: null +ray_scope: + desc: null + value: last +ddp_timeout: + desc: null + value: 1800 +torch_compile: + desc: null + value: false +torch_compile_backend: + desc: null + value: null +torch_compile_mode: + desc: null + value: null +dispatch_batches: + desc: null + value: null +split_batches: + desc: null + value: null +include_tokens_per_second: + desc: null + value: false +include_num_input_tokens_seen: + desc: null + value: false +neftune_noise_alpha: + desc: null + value: null +optim_target_modules: + desc: null + value: null diff --git a/wandb/run-20240411_040208-txpa3x4k/files/output.log b/wandb/run-20240411_040208-txpa3x4k/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4029897d060056a50ed8ca2419bf965b26501f1c --- /dev/null +++ b/wandb/run-20240411_040208-txpa3x4k/files/output.log @@ -0,0 +1,18 @@ +/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:557: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary. + warnings.warn(_create_warning_msg( +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' \ No newline at end of file diff --git a/wandb/run-20240411_040208-txpa3x4k/files/requirements.txt b/wandb/run-20240411_040208-txpa3x4k/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75d653a4d1ff18898bcd3ec2bda23c15dc7051d9 --- /dev/null +++ b/wandb/run-20240411_040208-txpa3x4k/files/requirements.txt @@ -0,0 +1,864 @@ +Babel==2.14.0 +Boruta==0.3 +Brotli==1.0.9 +CVXcanon==0.1.2 +Cartopy==0.22.0 +Cython==3.0.8 +Deprecated==1.2.14 +Farama-Notifications==0.0.4 +Flask==3.0.2 +Geohash==1.0 +GitPython==3.1.41 +ImageHash==4.3.1 +Janome==0.5.0 +Jinja2==3.1.2 +LunarCalendar==0.0.9 +Mako==1.3.2 +Markdown==3.5.2 +MarkupSafe==2.1.3 +MarkupSafe==2.1.5 +Pillow==9.5.0 +PuLP==2.8.0 +PyArabic==0.6.15 +PyJWT==2.8.0 +PyMeeus==0.5.12 +PySocks==1.7.1 +PyUpSet==0.1.1.post7 +PyWavelets==1.5.0 +PyYAML==6.0.1 +Pygments==2.17.2 +Pympler==1.0.1 +QtPy==2.4.1 +Rtree==1.2.0 +SQLAlchemy==2.0.25 +SecretStorage==3.3.3 +Send2Trash==1.8.2 +Shapely==1.8.5.post1 +Shimmy==1.3.0 +SimpleITK==2.3.1 +TPOT==0.12.1 +Theano-PyMC==1.1.2 +Theano==1.0.5 +Wand==0.6.13 +Werkzeug==3.0.2 +absl-py==1.4.0 +accelerate==0.28.0 +access==1.1.9 +affine==2.4.0 +aiobotocore==2.12.2 +aiofiles==22.1.0 +aiohttp-cors==0.7.0 +aiohttp==3.9.1 +aioitertools==0.11.0 +aiorwlock==1.3.0 +aiosignal==1.3.1 +aiosqlite==0.19.0 +albumentations==1.4.0 +alembic==1.13.1 +altair==5.3.0 +annotated-types==0.6.0 +annoy==1.17.3 +anyio==4.2.0 +apache-beam==2.46.0 +aplus==0.11.0 +appdirs==1.4.4 +archspec==0.2.3 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +array-record==0.5.0 +arrow==1.3.0 +arviz==0.17.1 +astroid==3.1.0 +astropy-iers-data==0.2024.4.1.0.33.14 +astropy==6.0.1 +asttokens==2.4.1 +astunparse==1.6.3 +async-lru==2.0.4 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +autopep8==2.0.4 +backoff==2.2.1 +bayesian-optimization==1.4.3 +beatrix_jupyterlab==2023.128.151533 +beautifulsoup4==4.12.2 +bitsandbytes==0.43.0 +blake3==0.2.1 +bleach==6.1.0 +blessed==1.20.0 +blinker==1.7.0 +blis==0.7.10 +blosc2==2.6.0 +bokeh==3.3.4 +boltons==23.1.1 +boto3==1.26.100 +botocore==1.34.51 +bq_helper==0.4.1 +bqplot==0.12.43 +branca==0.7.1 +brewer2mpl==1.4.1 +brotlipy==0.7.0 +cached-property==1.5.2 +cachetools==4.2.4 +cachetools==5.3.2 +catalogue==2.0.10 +catalyst==22.4 +catboost==1.2.3 +category-encoders==2.6.3 +certifi==2024.2.2 +cesium==0.12.1 +cffi==1.16.0 +charset-normalizer==3.3.2 +chex==0.1.86 +cleverhans==4.0.0 +click-plugins==1.1.1 +click==8.1.7 +cligj==0.7.2 +cloud-tpu-client==0.10 +cloud-tpu-profiler==2.4.0 +cloudpathlib==0.16.0 +cloudpickle==2.2.1 +cloudpickle==3.0.0 +cmdstanpy==1.2.2 +colorama==0.4.6 +colorcet==3.1.0 +colorful==0.5.6 +colorlog==6.8.2 +colorlover==0.3.0 +comm==0.2.1 +conda-libmamba-solver==23.7.0 +conda-package-handling==2.2.0 +conda==23.7.4 +conda_package_streaming==0.9.0 +confection==0.1.4 +contextily==1.6.0 +contourpy==1.2.0 +convertdate==2.4.0 +crcmod==1.7 +cryptography==41.0.7 +cuda-python==12.4.0 +cudf==23.8.0 +cufflinks==0.17.3 +cuml==23.8.0 +cupy==13.0.0 +cycler==0.12.1 +cymem==2.0.8 +cytoolz==0.12.3 +daal4py==2024.2.0 +daal==2024.2.0 +dacite==1.8.1 +dask-cuda==23.8.0 +dask-cudf==23.8.0 +dask-expr==1.0.9 +dask==2024.4.0 +dataclasses-json==0.6.4 +dataproc_jupyter_plugin==0.1.66 +datasets==2.16.0 +datashader==0.16.0 +datatile==1.0.3 +db-dtypes==1.2.0 +deap==1.4.1 +debugpy==1.8.0 +decorator==5.1.1 +deepdiff==6.7.1 +defusedxml==0.7.1 +deprecation==2.1.0 +descartes==1.1.0 +dill==0.3.7 +dipy==1.9.0 +distlib==0.3.8 +distributed==2023.7.1 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docker==7.0.0 +docopt==0.6.2 +docstring-parser==0.15 +docstring-to-markdown==0.15 +docutils==0.20.1 +earthengine-api==0.1.395 +easydict==1.13 +easyocr==1.7.1 +ecos==2.0.13 +eli5==0.13.0 +emoji==2.11.0 +en-core-web-lg==3.7.1 +en-core-web-sm==3.7.1 +entrypoints==0.4 +ephem==4.1.5 +esda==2.5.1 +essentia==2.1b6.dev1110 +et-xmlfile==1.1.0 +etils==1.6.0 +exceptiongroup==1.2.0 +executing==2.0.1 +explainable-ai-sdk==1.3.3 +fastai==2.7.14 +fastapi==0.108.0 +fastavro==1.9.3 +fastcore==1.5.29 +fastdownload==0.0.7 +fasteners==0.19 +fastjsonschema==2.19.1 +fastprogress==1.0.3 +fastrlock==0.8.2 +fasttext==0.9.2 +feather-format==0.4.1 +featuretools==1.30.0 +filelock==3.13.1 +fiona==1.9.6 +fitter==1.7.0 +flake8==7.0.0 +flashtext==2.7 +flatbuffers==23.5.26 +flax==0.8.2 +folium==0.16.0 +fonttools==4.47.0 +fonttools==4.50.0 +fqdn==1.5.1 +frozendict==2.4.1 +frozenlist==1.4.1 +fsspec==2023.10.0 +fsspec==2024.3.1 +funcy==2.0 +fury==0.10.0 +future==1.0.0 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gatspy==0.3 +gcsfs==2024.2.0 +gensim==4.3.2 +geographiclib==2.0 +geojson==3.1.0 +geopandas==0.14.3 +geoplot==0.5.1 +geopy==2.4.1 +geoviews==1.11.1 +ggplot==0.11.5 +giddy==2.3.5 +gitdb==4.0.11 +google-ai-generativelanguage==0.4.0 +google-api-core==2.11.1 +google-api-core==2.18.0 +google-api-python-client==2.125.0 +google-apitools==0.5.31 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-auth==2.26.1 +google-cloud-aiplatform==0.6.0a1 +google-cloud-artifact-registry==1.10.0 +google-cloud-automl==1.0.1 +google-cloud-bigquery==2.34.4 +google-cloud-bigtable==1.7.3 +google-cloud-core==2.4.1 +google-cloud-datastore==2.19.0 +google-cloud-dlp==3.14.0 +google-cloud-jupyter-config==0.0.5 +google-cloud-language==2.13.3 +google-cloud-monitoring==2.18.0 +google-cloud-pubsub==2.19.0 +google-cloud-pubsublite==1.9.0 +google-cloud-recommendations-ai==0.7.1 +google-cloud-resource-manager==1.11.0 +google-cloud-spanner==3.40.1 +google-cloud-storage==1.44.0 +google-cloud-translate==3.12.1 +google-cloud-videointelligence==2.13.3 +google-cloud-vision==2.8.0 +google-crc32c==1.5.0 +google-generativeai==0.4.1 +google-pasta==0.2.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 +gplearn==0.4.2 +gpustat==1.0.0 +gpxpy==1.6.2 +graphviz==0.20.3 +greenlet==3.0.3 +grpc-google-iam-v1==0.12.7 +grpcio-status==1.48.1 +grpcio-status==1.48.2 +grpcio==1.51.1 +grpcio==1.60.0 +gviz-api==1.10.0 +gym-notices==0.0.8 +gym==0.26.2 +gymnasium==0.29.0 +h11==0.14.0 +h2o==3.46.0.1 +h5netcdf==1.3.0 +h5py==3.10.0 +haversine==2.8.1 +hdfs==2.7.3 +hep-ml==0.7.2 +hijri-converter==2.3.1 +hmmlearn==0.3.2 +holidays==0.24 +holoviews==1.18.3 +hpsklearn==0.1.0 +html5lib==1.1 +htmlmin==0.1.12 +httpcore==1.0.5 +httplib2==0.21.0 +httptools==0.6.1 +httpx==0.27.0 +huggingface-hub==0.22.2 +hunspell==0.5.5 +hydra-slayer==0.5.0 +hyperopt==0.2.7 +hypertools==0.8.0 +idna==3.6 +igraph==0.11.4 +imagecodecs==2024.1.1 +imageio==2.33.1 +imbalanced-learn==0.12.2 +imgaug==0.4.0 +importlib-metadata==6.11.0 +importlib-metadata==7.0.1 +importlib-resources==6.1.1 +inequality==1.0.1 +iniconfig==2.0.0 +ipydatawidgets==4.3.5 +ipykernel==6.28.0 +ipyleaflet==0.18.2 +ipympl==0.7.0 +ipython-genutils==0.2.0 +ipython-genutils==0.2.0 +ipython-sql==0.5.0 +ipython==8.20.0 +ipyvolume==0.6.3 +ipyvue==1.10.2 +ipyvuetify==1.9.3 +ipywebrtc==0.6.0 +ipywidgets==7.7.1 +isoduration==20.11.0 +isort==5.13.2 +isoweek==1.3.3 +itsdangerous==2.1.2 +jaraco.classes==3.3.0 +jax-jumpy==1.0.0 +jax==0.4.23 +jaxlib==0.4.23.dev20240116 +jedi==0.19.1 +jeepney==0.8.0 +jieba==0.42.1 +jmespath==1.0.1 +joblib==1.3.2 +json5==0.9.14 +jsonpatch==1.33 +jsonpointer==2.4 +jsonschema-specifications==2023.12.1 +jsonschema==4.20.0 +jupyter-console==6.6.3 +jupyter-events==0.9.0 +jupyter-http-over-ws==0.0.8 +jupyter-lsp==1.5.1 +jupyter-server-mathjax==0.2.6 +jupyter-ydoc==0.2.5 +jupyter_client==7.4.9 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +jupyter_server==2.13.0 +jupyter_server_fileid==0.9.1 +jupyter_server_proxy==4.1.0 +jupyter_server_terminals==0.5.1 +jupyter_server_ydoc==0.8.0 +jupyterlab-lsp==5.1.0 +jupyterlab-widgets==3.0.9 +jupyterlab==4.1.5 +jupyterlab_git==0.44.0 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.25.2 +jupytext==1.16.0 +kaggle-environments==1.14.3 +kaggle==1.6.8 +kagglehub==0.2.2 +keras-cv==0.8.2 +keras-nlp==0.8.2 +keras-tuner==1.4.6 +keras==3.1.1 +kernels-mixer==0.0.7 +keyring==24.3.0 +keyrings.google-artifactregistry-auth==1.1.2 +kfp-pipeline-spec==0.2.2 +kfp-server-api==2.0.5 +kfp==2.5.0 +kiwisolver==1.4.5 +kmapper==2.0.1 +kmodes==0.12.2 +korean-lunar-calendar==0.3.1 +kornia==0.7.2 +kornia_rs==0.1.3 +kt-legacy==1.0.5 +kubernetes==26.1.0 +langcodes==3.3.0 +langid==1.1.6 +lazy_loader==0.3 +learntools==0.3.4 +leven==1.0.4 +libclang==16.0.6 +libmambapy==1.5.0 +libpysal==4.9.2 +librosa==0.10.1 +lightgbm==4.2.0 +lightning-utilities==0.11.2 +lime==0.2.0.1 +line-profiler==4.1.2 +linkify-it-py==2.0.3 +llvmlite==0.41.1 +llvmlite==0.42.0 +lml==0.1.0 +locket==1.0.0 +loguru==0.7.2 +lxml==5.2.1 +lz4==4.3.3 +mamba==1.5.0 +mapclassify==2.6.1 +markdown-it-py==3.0.0 +marshmallow==3.21.1 +matplotlib-inline==0.1.6 +matplotlib-venn==0.11.10 +matplotlib==3.7.5 +matplotlib==3.8.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +memory-profiler==0.61.0 +menuinst==2.0.1 +mercantile==1.2.1 +mgwr==2.2.1 +missingno==0.5.2 +mistune==0.8.4 +mizani==0.11.1 +ml-dtypes==0.2.0 +mlcrate==0.2.0 +mlens==0.2.3 +mlxtend==0.23.1 +mne==1.6.1 +mnist==0.2.2 +momepy==0.7.0 +more-itertools==10.2.0 +mpld3==0.5.10 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multimethod==1.10 +multipledispatch==1.0.0 +multiprocess==0.70.15 +munkres==1.1.4 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +namex==0.0.7 +nb-conda-kernels==2.3.1 +nb_conda==2.2.1 +nbclassic==1.0.0 +nbclient==0.5.13 +nbconvert==6.4.5 +nbdime==3.2.0 +nbformat==5.9.2 +ndindex==1.8 +nest-asyncio==1.5.8 +networkx==3.2.1 +nibabel==5.2.1 +nilearn==0.10.3 +ninja==1.11.1.1 +nltk==3.2.4 +nose==1.3.7 +notebook==6.5.4 +notebook==6.5.6 +notebook_executor==0.2 +notebook_shim==0.2.3 +numba==0.58.1 +numba==0.59.1 +numexpr==2.10.0 +numpy==1.26.4 +nvidia-ml-py==11.495.46 +nvtx==0.2.10 +oauth2client==4.1.3 +oauthlib==3.2.2 +objsize==0.6.1 +odfpy==1.4.1 +olefile==0.47 +onnx==1.16.0 +opencensus-context==0.1.3 +opencensus==0.11.4 +opencv-contrib-python==4.9.0.80 +opencv-python-headless==4.9.0.80 +opencv-python==4.9.0.80 +openpyxl==3.1.2 +openslide-python==1.3.1 +opentelemetry-api==1.22.0 +opentelemetry-exporter-otlp-proto-common==1.22.0 +opentelemetry-exporter-otlp-proto-grpc==1.22.0 +opentelemetry-exporter-otlp-proto-http==1.22.0 +opentelemetry-exporter-otlp==1.22.0 +opentelemetry-proto==1.22.0 +opentelemetry-sdk==1.22.0 +opentelemetry-semantic-conventions==0.43b0 +opt-einsum==3.3.0 +optax==0.2.2 +optree==0.11.0 +optuna==3.6.1 +orbax-checkpoint==0.5.7 +ordered-set==4.1.0 +orjson==3.9.10 +ortools==9.4.1874 +osmnx==1.9.2 +overrides==7.4.0 +packaging==21.3 +pandas-datareader==0.10.0 +pandas-profiling==3.6.6 +pandas-summary==0.2.0 +pandas==2.1.4 +pandas==2.2.1 +pandasql==0.7.3 +pandocfilters==1.5.0 +panel==1.3.8 +papermill==2.5.0 +param==2.1.0 +parso==0.8.3 +partd==1.4.1 +path.py==12.5.0 +path==16.10.0 +pathos==0.3.2 +pathy==0.10.3 +patsy==0.5.6 +pdf2image==1.17.0 +peft==0.10.0 +pettingzoo==1.24.0 +pexpect==4.8.0 +pexpect==4.9.0 +phik==0.12.4 +pickleshare==0.7.5 +pillow==10.3.0 +pip==23.3.2 +pkgutil_resolve_name==1.3.10 +platformdirs==4.2.0 +plotly-express==0.4.1 +plotly==5.18.0 +plotnine==0.13.4 +pluggy==1.4.0 +pointpats==2.4.0 +polars==0.20.18 +polyglot==16.7.4 +pooch==1.8.1 +pox==0.3.4 +ppca==0.0.4 +ppft==1.7.6.8 +preprocessing==0.1.13 +preshed==3.0.9 +prettytable==3.9.0 +progressbar2==4.4.2 +prometheus-client==0.19.0 +promise==2.3 +prompt-toolkit==3.0.42 +prompt-toolkit==3.0.43 +prophet==1.1.1 +proto-plus==1.23.0 +protobuf==3.20.3 +protobuf==4.21.12 +psutil==5.9.3 +psutil==5.9.7 +ptyprocess==0.7.0 +pudb==2024.1 +pure-eval==0.2.2 +py-cpuinfo==9.0.0 +py-spy==0.3.14 +py4j==0.10.9.7 +pyLDAvis==3.4.1 +pyOpenSSL==23.3.0 +pyaml==23.12.0 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11==2.12.0 +pyclipper==1.3.0.post5 +pycodestyle==2.11.1 +pycosat==0.6.6 +pycparser==2.21 +pycryptodome==3.20.0 +pyct==0.5.0 +pycuda==2024.1 +pydantic==2.5.3 +pydantic==2.6.4 +pydantic_core==2.14.6 +pydantic_core==2.16.3 +pydegensac==0.1.2 +pydicom==2.4.4 +pydocstyle==6.3.0 +pydot==1.4.2 +pydub==0.25.1 +pyemd==1.0.0 +pyerfa==2.0.1.1 +pyexcel-io==0.6.6 +pyexcel-ods==0.6.0 +pyflakes==3.2.0 +pygltflib==1.16.2 +pykalman==0.9.7 +pylibraft==23.8.0 +pylint==3.1.0 +pymc3==3.11.4 +pymongo==3.13.0 +pynndescent==0.5.12 +pynvml==11.4.1 +pynvrtc==9.2 +pyparsing==3.1.1 +pyparsing==3.1.2 +pypdf==4.1.0 +pyproj==3.6.1 +pysal==24.1 +pyshp==2.3.1 +pytesseract==0.3.10 +pytest==8.1.1 +python-bidi==0.4.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.0 +python-json-logger==2.0.7 +python-louvain==0.16 +python-lsp-jsonrpc==1.1.2 +python-lsp-server==1.11.0 +python-slugify==8.0.4 +python-utils==3.8.2 +pythreejs==2.4.2 +pytoolconfig==1.3.1 +pytools==2024.1.1 +pytorch-ignite==0.5.0.post2 +pytorch-lightning==2.2.1 +pytz==2023.3.post1 +pytz==2024.1 +pyu2f==0.1.5 +pyviz_comms==3.0.2 +pyzmq==24.0.1 +pyzmq==25.1.2 +qgrid==1.3.1 +qtconsole==5.5.1 +quantecon==0.7.2 +qudida==0.0.4 +raft-dask==23.8.0 +rasterio==1.3.9 +rasterstats==0.19.0 +ray-cpp==2.9.0 +ray==2.9.0 +referencing==0.32.1 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==0.10.1 +requests==2.31.0 +retrying==1.3.3 +retrying==1.3.4 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rgf-python==3.12.0 +rich-click==1.7.4 +rich==13.7.0 +rich==13.7.1 +rmm==23.8.0 +rope==1.13.0 +rpds-py==0.16.2 +rsa==4.9 +ruamel-yaml-conda==0.15.100 +ruamel.yaml.clib==0.2.7 +ruamel.yaml==0.17.40 +s2sphere==0.2.5 +s3fs==2024.2.0 +s3transfer==0.6.2 +safetensors==0.4.2 +scattertext==0.1.19 +scikit-image==0.22.0 +scikit-learn-intelex==2024.2.0 +scikit-learn==1.2.2 +scikit-multilearn==0.2.0 +scikit-optimize==0.10.1 +scikit-plot==0.3.7 +scikit-surprise==1.1.3 +scipy==1.11.4 +scipy==1.12.0 +seaborn==0.12.2 +segment_anything==1.0 +segregation==2.5 +semver==3.0.2 +sentencepiece==0.2.0 +sentry-sdk==1.44.1 +setproctitle==1.3.3 +setuptools-git==1.2 +setuptools-scm==8.0.4 +setuptools==69.0.3 +shap==0.44.1 +shapely==2.0.3 +shellingham==1.5.4 +shtab==1.7.1 +simpervisor==1.0.0 +simplejson==3.19.2 +six==1.16.0 +sklearn-pandas==2.2.0 +slicer==0.0.7 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +spaghetti==1.7.5.post1 +spectral==0.23.1 +spglm==1.1.0 +sphinx-rtd-theme==0.2.4 +spint==1.0.7 +splot==1.1.5.post1 +spopt==0.6.0 +spreg==1.4.2 +spvcm==0.3.0 +sqlparse==0.4.4 +squarify==0.4.3 +srsly==2.4.8 +stable-baselines3==2.1.0 +stack-data==0.6.2 +stack-data==0.6.3 +stanio==0.5.0 +starlette==0.32.0.post1 +statsmodels==0.14.1 +stemming==1.0.1 +stop-words==2018.7.23 +stopit==1.1.2 +stumpy==1.12.0 +sympy==1.12 +tables==3.9.2 +tabulate==0.9.0 +tangled-up-in-unicode==0.2.0 +tbb==2021.12.0 +tblib==3.0.0 +tenacity==8.2.3 +tensorboard-data-server==0.7.2 +tensorboard-plugin-profile==2.15.0 +tensorboard==2.15.1 +tensorboardX==2.6.2.2 +tensorflow-cloud==0.1.16 +tensorflow-datasets==4.9.4 +tensorflow-decision-forests==1.8.1 +tensorflow-estimator==2.15.0 +tensorflow-hub==0.16.1 +tensorflow-io-gcs-filesystem==0.35.0 +tensorflow-io==0.35.0 +tensorflow-metadata==0.14.0 +tensorflow-probability==0.23.0 +tensorflow-serving-api==2.14.1 +tensorflow-text==2.15.0 +tensorflow-transform==0.14.0 +tensorflow==2.15.0 +tensorstore==0.1.56 +termcolor==2.4.0 +terminado==0.18.0 +testpath==0.6.0 +text-unidecode==1.3 +textblob==0.18.0.post0 +texttable==1.7.0 +tf_keras==2.15.1 +tfp-nightly==0.24.0.dev0 +thinc==8.2.2 +threadpoolctl==3.2.0 +tifffile==2023.12.9 +timm==0.9.16 +tinycss2==1.2.1 +tobler==0.11.2 +tokenizers==0.15.2 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.12.4 +toolz==0.12.1 +torch==2.1.2 +torchaudio==2.1.2 +torchdata==0.7.1 +torchinfo==1.8.0 +torchmetrics==1.3.2 +torchtext==0.16.2 +torchvision==0.16.2 +tornado==6.3.3 +tqdm==4.66.1 +traceml==1.0.8 +traitlets==5.9.0 +traittypes==0.2.1 +transformers==4.39.3 +treelite-runtime==3.2.0 +treelite==3.2.0 +trl==0.8.1 +truststore==0.8.0 +trx-python==0.2.9 +tsfresh==0.20.2 +typeguard==4.1.5 +typer==0.9.0 +typer==0.9.4 +types-python-dateutil==2.8.19.20240106 +typing-inspect==0.9.0 +typing-utils==0.1.0 +typing_extensions==4.9.0 +tyro==0.8.3 +tzdata==2023.4 +uc-micro-py==1.0.3 +ucx-py==0.33.0 +ujson==5.9.0 +umap-learn==0.5.5 +unicodedata2==15.1.0 +update-checker==0.18.0 +uri-template==1.3.0 +uritemplate==3.0.1 +urllib3==1.26.18 +urllib3==2.1.0 +urwid==2.6.10 +urwid_readline==0.14 +uvicorn==0.25.0 +uvloop==0.19.0 +vaex-astro==0.9.3 +vaex-core==4.17.1 +vaex-hdf5==0.14.1 +vaex-jupyter==0.8.2 +vaex-ml==0.18.3 +vaex-server==0.9.0 +vaex-viz==0.5.4 +vaex==4.17.0 +vec_noise==1.1.4 +vecstack==0.4.0 +virtualenv==20.21.0 +visions==0.7.5 +vowpalwabbit==9.9.0 +vtk==9.3.0 +wandb==0.16.5 +wasabi==1.1.2 +watchfiles==0.21.0 +wavio==0.0.8 +wcwidth==0.2.13 +weasel==0.3.4 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.7.0 +websockets==12.0 +wfdb==4.1.2 +whatthepatch==1.0.5 +wheel==0.42.0 +widgetsnbextension==3.6.6 +witwidget==1.8.1 +woodwork==0.29.0 +wordcloud==1.9.3 +wordsegment==1.3.1 +wrapt==1.14.1 +xarray-einstats==0.7.0 +xarray==2024.3.0 +xgboost==2.0.3 +xvfbwrapper==0.2.9 +xxhash==3.4.1 +xyzservices==2023.10.1 +y-py==0.6.2 +yapf==0.40.2 +yarl==1.9.3 +yarl==1.9.4 +ydata-profiling==4.6.4 +yellowbrick==1.5 +ypy-websocket==0.8.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.22.0 \ No newline at end of file diff --git a/wandb/run-20240411_040208-txpa3x4k/files/wandb-metadata.json b/wandb/run-20240411_040208-txpa3x4k/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..26f8c76971e1b36e4ba1cd4a0e1b8e4dd0c81062 --- /dev/null +++ b/wandb/run-20240411_040208-txpa3x4k/files/wandb-metadata.json @@ -0,0 +1,66 @@ +{ + "os": "Linux-5.15.133+-x86_64-with-glibc2.31", + "python": "3.10.13", + "heartbeatAt": "2024-04-11T04:02:08.804959", + "startedAt": "2024-04-11T04:02:08.129415", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "kaggle.ipynb", + "codePathLocal": null, + "root": "/kaggle/working", + "host": "321ddc84c049", + "username": "root", + "executable": "/opt/conda/bin/python3.10", + "cpu_count": 2, + "cpu_count_logical": 4, + "cpu_freq": { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 8062.387607574463, + "used": 5566.961982727051 + } + }, + "gpu": "Tesla T4", + "gpu_count": 2, + "gpu_devices": [ + { + "name": "Tesla T4", + "memory_total": 16106127360 + }, + { + "name": "Tesla T4", + "memory_total": 16106127360 + } + ], + "memory": { + "total": 31.357559204101562 + } +} diff --git a/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json b/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..87c5aa73d996e17cbdc640c739aa3ccb45063765 --- /dev/null +++ b/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss": 27.1232, "train/grad_norm": 212.8720245361328, "train/learning_rate": 5.7142857142857145e-06, "train/epoch": 0.0, "train/global_step": 40, "_timestamp": 1712808237.341076, "_runtime": 109.20561385154724, "_step": 1, "_wandb": {"runtime": 120}} \ No newline at end of file diff --git a/wandb/run-20240411_040208-txpa3x4k/logs/debug-internal.log b/wandb/run-20240411_040208-txpa3x4k/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fc9e840484661972d567a3c3814902aee9d692f0 --- /dev/null +++ b/wandb/run-20240411_040208-txpa3x4k/logs/debug-internal.log @@ -0,0 +1,273 @@ +2024-04-11 04:02:08,136 INFO StreamThr :455 [internal.py:wandb_internal():86] W&B internal server running at pid: 455, started at: 2024-04-11 04:02:08.135570 +2024-04-11 04:02:08,137 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status +2024-04-11 04:02:08,532 INFO WriterThread:455 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/run-txpa3x4k.wandb +2024-04-11 04:02:08,532 DEBUG SenderThread:455 [sender.py:send():379] send: header +2024-04-11 04:02:08,535 DEBUG SenderThread:455 [sender.py:send():379] send: run +2024-04-11 04:02:08,682 INFO SenderThread:455 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files +2024-04-11 04:02:08,683 INFO SenderThread:455 [sender.py:_start_run_threads():1124] run started: txpa3x4k with start time 1712808128.135462 +2024-04-11 04:02:08,691 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: check_version +2024-04-11 04:02:08,691 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: check_version +2024-04-11 04:02:08,778 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: run_start +2024-04-11 04:02:08,790 DEBUG HandlerThread:455 [system_info.py:__init__():26] System info init +2024-04-11 04:02:08,790 DEBUG HandlerThread:455 [system_info.py:__init__():41] System info init done +2024-04-11 04:02:08,790 INFO HandlerThread:455 [system_monitor.py:start():194] Starting system monitor +2024-04-11 04:02:08,790 INFO SystemMonitor:455 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-11 04:02:08,791 INFO SystemMonitor:455 [interfaces.py:start():190] Started cpu monitoring +2024-04-11 04:02:08,791 INFO SystemMonitor:455 [interfaces.py:start():190] Started disk monitoring +2024-04-11 04:02:08,792 INFO HandlerThread:455 [system_monitor.py:probe():214] Collecting system info +2024-04-11 04:02:08,792 INFO SystemMonitor:455 [interfaces.py:start():190] Started gpu monitoring +2024-04-11 04:02:08,793 INFO SystemMonitor:455 [interfaces.py:start():190] Started memory monitoring +2024-04-11 04:02:08,793 INFO SystemMonitor:455 [interfaces.py:start():190] Started network monitoring +2024-04-11 04:02:08,804 DEBUG HandlerThread:455 [system_info.py:probe():150] Probing system +2024-04-11 04:02:08,807 DEBUG HandlerThread:455 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-11 04:02:08,807 DEBUG HandlerThread:455 [system_info.py:probe():198] Probing system done +2024-04-11 04:02:08,807 DEBUG HandlerThread:455 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-11T04:02:08.804959', 'startedAt': '2024-04-11T04:02:08.129415', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '321ddc84c049', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.19, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5566.961982727051}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-04-11 04:02:08,807 INFO HandlerThread:455 [system_monitor.py:probe():224] Finished collecting system info +2024-04-11 04:02:08,807 INFO HandlerThread:455 [system_monitor.py:probe():227] Publishing system info +2024-04-11 04:02:08,807 DEBUG HandlerThread:455 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-11 04:02:09,685 INFO Thread-12 :455 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/conda-environment.yaml +2024-04-11 04:02:23,823 ERROR HandlerThread:455 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +Traceback (most recent call last): + File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda + subprocess.call( + File "/opt/conda/lib/python3.10/subprocess.py", line 347, in call + return p.wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1209, in wait + return self._wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait + raise TimeoutExpired(self.args, timeout) +subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-04-11 04:02:23,824 DEBUG HandlerThread:455 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-11 04:02:23,825 INFO HandlerThread:455 [system_monitor.py:probe():229] Finished publishing system info +2024-04-11 04:02:23,830 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:23,830 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:02:23,830 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:23,830 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:02:23,830 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:23,830 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:02:23,831 DEBUG SenderThread:455 [sender.py:send():379] send: files +2024-04-11 04:02:23,831 INFO SenderThread:455 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-11 04:02:24,033 INFO wandb-upload_0:455 [upload_job.py:push():131] Uploaded file /tmp/tmp1s3x48hcwandb/4qm5hpvk-wandb-metadata.json +2024-04-11 04:02:24,688 INFO Thread-12 :455 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/wandb-metadata.json +2024-04-11 04:02:24,755 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: python_packages +2024-04-11 04:02:24,756 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: python_packages +2024-04-11 04:02:24,757 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:02:24,758 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:02:24,795 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:02:24,805 DEBUG SenderThread:455 [sender.py:send():379] send: telemetry +2024-04-11 04:02:24,805 DEBUG SenderThread:455 [sender.py:send():379] send: config +2024-04-11 04:02:24,807 DEBUG SenderThread:455 [sender.py:send():379] send: metric +2024-04-11 04:02:24,808 DEBUG SenderThread:455 [sender.py:send():379] send: telemetry +2024-04-11 04:02:24,808 DEBUG SenderThread:455 [sender.py:send():379] send: metric +2024-04-11 04:02:24,808 WARNING SenderThread:455 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-11 04:02:24,808 DEBUG SenderThread:455 [sender.py:send():379] send: telemetry +2024-04-11 04:02:25,688 INFO Thread-12 :455 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:02:25,689 INFO Thread-12 :455 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/requirements.txt +2024-04-11 04:02:27,689 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:02:29,171 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:29,690 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:02:34,172 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:39,178 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:39,693 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/config.yaml +2024-04-11 04:02:40,355 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:02:40,355 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:02:40,358 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:02:44,408 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:49,409 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:54,410 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:02:55,351 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:02:55,352 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:02:55,392 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:03:00,394 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:05,395 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:08,794 DEBUG SystemMonitor:455 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-11 04:03:08,795 DEBUG SenderThread:455 [sender.py:send():379] send: stats +2024-04-11 04:03:10,353 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:03:10,353 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:03:10,394 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:03:10,420 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:12,619 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:03:12,622 DEBUG SenderThread:455 [sender.py:send():379] send: metric +2024-04-11 04:03:12,623 DEBUG SenderThread:455 [sender.py:send():379] send: metric +2024-04-11 04:03:12,623 DEBUG SenderThread:455 [sender.py:send():379] send: metric +2024-04-11 04:03:12,623 DEBUG SenderThread:455 [sender.py:send():379] send: metric +2024-04-11 04:03:12,623 DEBUG SenderThread:455 [sender.py:send():379] send: history +2024-04-11 04:03:12,623 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:03:12,623 INFO SenderThread:455 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:03:12,709 INFO Thread-12 :455 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json +2024-04-11 04:03:15,592 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:15,710 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:03:17,711 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:03:20,730 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:25,368 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:03:25,368 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:03:25,409 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:03:26,469 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:31,470 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:36,471 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:38,796 DEBUG SenderThread:455 [sender.py:send():379] send: stats +2024-04-11 04:03:40,358 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:03:40,358 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:03:40,359 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:03:42,393 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:42,721 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/config.yaml +2024-04-11 04:03:47,526 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:52,527 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:55,357 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:03:55,358 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:03:55,359 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:03:57,341 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:03:57,342 DEBUG SenderThread:455 [sender.py:send():379] send: history +2024-04-11 04:03:57,343 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:03:57,345 INFO SenderThread:455 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:03:57,727 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json +2024-04-11 04:03:58,374 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:03:59,728 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:04:01,729 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:04:03,528 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:04:03,729 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:04:08,529 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:04:08,797 DEBUG SenderThread:455 [sender.py:send():379] send: stats +2024-04-11 04:04:09,554 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: pause +2024-04-11 04:04:09,554 INFO HandlerThread:455 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-11 04:04:09,554 INFO HandlerThread:455 [system_monitor.py:finish():203] Stopping system monitor +2024-04-11 04:04:09,554 DEBUG SystemMonitor:455 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-11 04:04:09,554 DEBUG SystemMonitor:455 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-11 04:04:09,555 INFO HandlerThread:455 [interfaces.py:finish():202] Joined cpu monitor +2024-04-11 04:04:09,555 INFO HandlerThread:455 [interfaces.py:finish():202] Joined disk monitor +2024-04-11 04:04:09,567 INFO HandlerThread:455 [interfaces.py:finish():202] Joined gpu monitor +2024-04-11 04:04:09,567 INFO HandlerThread:455 [interfaces.py:finish():202] Joined memory monitor +2024-04-11 04:04:09,567 INFO HandlerThread:455 [interfaces.py:finish():202] Joined network monitor +2024-04-11 04:04:09,568 DEBUG SenderThread:455 [sender.py:send():379] send: stats +2024-04-11 04:04:10,358 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:04:10,358 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:04:10,359 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:04:14,201 DEBUG SenderThread:455 [sender.py:send():379] send: exit +2024-04-11 04:04:14,201 INFO SenderThread:455 [sender.py:send_exit():586] handling exit code: 0 +2024-04-11 04:04:14,201 INFO SenderThread:455 [sender.py:send_exit():588] handling runtime: 120 +2024-04-11 04:04:14,201 INFO SenderThread:455 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:04:14,202 INFO SenderThread:455 [sender.py:send_exit():594] send defer +2024-04-11 04:04:14,202 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,202 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 0 +2024-04-11 04:04:14,202 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:04:14,202 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,202 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-04-11 04:04:14,202 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 1 +2024-04-11 04:04:14,203 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,203 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 1 +2024-04-11 04:04:14,203 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,203 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-04-11 04:04:14,203 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 2 +2024-04-11 04:04:14,203 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,203 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 2 +2024-04-11 04:04:14,203 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,204 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-04-11 04:04:14,204 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 3 +2024-04-11 04:04:14,204 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,204 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 3 +2024-04-11 04:04:14,204 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,204 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-04-11 04:04:14,204 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 4 +2024-04-11 04:04:14,204 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,204 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 4 +2024-04-11 04:04:14,204 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,205 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-04-11 04:04:14,205 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 5 +2024-04-11 04:04:14,205 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,205 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 5 +2024-04-11 04:04:14,205 DEBUG SenderThread:455 [sender.py:send():379] send: summary +2024-04-11 04:04:14,206 INFO SenderThread:455 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:04:14,206 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,206 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-04-11 04:04:14,206 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 6 +2024-04-11 04:04:14,206 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,206 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 6 +2024-04-11 04:04:14,206 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,206 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-04-11 04:04:14,206 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 7 +2024-04-11 04:04:14,207 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:04:14,207 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,207 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 7 +2024-04-11 04:04:14,207 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,207 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-04-11 04:04:14,733 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json +2024-04-11 04:04:14,942 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 8 +2024-04-11 04:04:14,942 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,943 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 8 +2024-04-11 04:04:14,943 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,943 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-04-11 04:04:14,943 INFO SenderThread:455 [job_builder.py:build():318] Attempting to build job artifact +2024-04-11 04:04:14,945 INFO SenderThread:455 [job_builder.py:_get_source_type():466] no source found +2024-04-11 04:04:14,945 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 9 +2024-04-11 04:04:14,945 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:14,945 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 9 +2024-04-11 04:04:14,945 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:14,946 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-04-11 04:04:14,946 INFO SenderThread:455 [dir_watcher.py:finish():358] shutting down directory watcher +2024-04-11 04:04:15,201 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:04:15,734 INFO Thread-12 :455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:04:15,735 INFO SenderThread:455 [dir_watcher.py:finish():388] scan: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files +2024-04-11 04:04:15,735 INFO SenderThread:455 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/config.yaml config.yaml +2024-04-11 04:04:15,735 INFO SenderThread:455 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/requirements.txt requirements.txt +2024-04-11 04:04:15,738 INFO SenderThread:455 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/conda-environment.yaml conda-environment.yaml +2024-04-11 04:04:15,739 INFO SenderThread:455 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json wandb-summary.json +2024-04-11 04:04:15,739 INFO SenderThread:455 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/wandb-metadata.json wandb-metadata.json +2024-04-11 04:04:15,739 INFO SenderThread:455 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log output.log +2024-04-11 04:04:15,742 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 10 +2024-04-11 04:04:15,742 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:04:15,743 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:15,743 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 10 +2024-04-11 04:04:15,749 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:15,749 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-04-11 04:04:15,749 INFO SenderThread:455 [file_pusher.py:finish():172] shutting down file pusher +2024-04-11 04:04:15,892 INFO wandb-upload_0:455 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/config.yaml +2024-04-11 04:04:15,944 INFO wandb-upload_1:455 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/requirements.txt +2024-04-11 04:04:15,965 INFO wandb-upload_3:455 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/output.log +2024-04-11 04:04:15,966 INFO wandb-upload_2:455 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040208-txpa3x4k/files/wandb-summary.json +2024-04-11 04:04:16,167 INFO Thread-11 (_thread_body):455 [sender.py:transition_state():614] send defer: 11 +2024-04-11 04:04:16,167 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:16,168 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 11 +2024-04-11 04:04:16,168 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:16,168 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-04-11 04:04:16,168 INFO SenderThread:455 [file_pusher.py:join():178] waiting for file pusher +2024-04-11 04:04:16,169 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 12 +2024-04-11 04:04:16,169 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:16,169 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 12 +2024-04-11 04:04:16,169 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:16,169 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-04-11 04:04:16,169 INFO SenderThread:455 [file_stream.py:finish():614] file stream finish called +2024-04-11 04:04:16,202 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:04:16,257 INFO SenderThread:455 [file_stream.py:finish():618] file stream finish is done +2024-04-11 04:04:16,257 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 13 +2024-04-11 04:04:16,257 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:04:16,258 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:16,258 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 13 +2024-04-11 04:04:16,258 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:16,258 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-04-11 04:04:16,258 INFO SenderThread:455 [sender.py:transition_state():614] send defer: 14 +2024-04-11 04:04:16,258 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:04:16,259 INFO HandlerThread:455 [handler.py:handle_request_defer():172] handle defer: 14 +2024-04-11 04:04:16,259 DEBUG SenderThread:455 [sender.py:send():379] send: final +2024-04-11 04:04:16,259 DEBUG SenderThread:455 [sender.py:send():379] send: footer +2024-04-11 04:04:16,259 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: defer +2024-04-11 04:04:16,259 INFO SenderThread:455 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-04-11 04:04:16,260 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:04:16,260 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:04:16,261 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:04:16,261 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:04:16,262 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: server_info +2024-04-11 04:04:16,262 DEBUG SenderThread:455 [sender.py:send_request():406] send_request: server_info +2024-04-11 04:04:16,265 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: get_summary +2024-04-11 04:04:16,266 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: sampled_history +2024-04-11 04:04:16,267 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:04:16,277 INFO MainThread:455 [wandb_run.py:_footer_history_summary_info():3920] rendering history +2024-04-11 04:04:16,277 INFO MainThread:455 [wandb_run.py:_footer_history_summary_info():3952] rendering summary +2024-04-11 04:04:16,278 INFO MainThread:455 [wandb_run.py:_footer_sync_info():3879] logging synced files +2024-04-11 04:04:16,278 DEBUG HandlerThread:455 [handler.py:handle_request():146] handle_request: shutdown +2024-04-11 04:04:16,278 INFO HandlerThread:455 [handler.py:finish():866] shutting down handler +2024-04-11 04:04:17,262 INFO WriterThread:455 [datastore.py:close():296] close: /kaggle/working/wandb/run-20240411_040208-txpa3x4k/run-txpa3x4k.wandb +2024-04-11 04:04:17,267 WARNING StreamThr :455 [internal.py:is_dead():414] Internal process exiting, parent pid 414 disappeared +2024-04-11 04:04:17,268 ERROR StreamThr :455 [internal.py:wandb_internal():152] Internal process shutdown. +2024-04-11 04:04:17,277 INFO SenderThread:455 [sender.py:finish():1546] shutting down sender +2024-04-11 04:04:17,277 INFO SenderThread:455 [file_pusher.py:finish():172] shutting down file pusher +2024-04-11 04:04:17,277 INFO SenderThread:455 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240411_040208-txpa3x4k/logs/debug.log b/wandb/run-20240411_040208-txpa3x4k/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7080f20b9b0ff179aa6e5e5d75d9cfd07fa54522 --- /dev/null +++ b/wandb/run-20240411_040208-txpa3x4k/logs/debug.log @@ -0,0 +1,33 @@ +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Current SDK version is 0.16.5 +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Configure stats pid to 414 +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Loading settings from /kaggle/working/wandb/settings +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': ''} +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_init.py:_log_setup():527] Logging user logs to /kaggle/working/wandb/run-20240411_040208-txpa3x4k/logs/debug.log +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_init.py:_log_setup():528] Logging internal logs to /kaggle/working/wandb/run-20240411_040208-txpa3x4k/logs/debug-internal.log +2024-04-11 04:02:08,131 INFO MainThread:414 [wandb_init.py:_jupyter_setup():473] configuring jupyter hooks +2024-04-11 04:02:08,132 INFO MainThread:414 [wandb_init.py:init():567] calling init triggers +2024-04-11 04:02:08,132 INFO MainThread:414 [wandb_init.py:init():574] wandb.init called with sweep_config: {} +config: {} +2024-04-11 04:02:08,132 INFO MainThread:414 [wandb_init.py:init():617] starting backend +2024-04-11 04:02:08,132 INFO MainThread:414 [wandb_init.py:init():621] setting up manager +2024-04-11 04:02:08,134 INFO MainThread:414 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-04-11 04:02:08,135 INFO MainThread:414 [wandb_init.py:init():629] backend started and connected +2024-04-11 04:02:08,148 INFO MainThread:414 [wandb_run.py:_label_probe_notebook():1299] probe notebook +2024-04-11 04:02:08,531 INFO MainThread:414 [wandb_init.py:init():721] updated telemetry +2024-04-11 04:02:08,534 INFO MainThread:414 [wandb_init.py:init():754] communicating run to backend with 90.0 second timeout +2024-04-11 04:02:08,691 INFO MainThread:414 [wandb_run.py:_on_init():2344] communicating current version +2024-04-11 04:02:08,772 INFO MainThread:414 [wandb_run.py:_on_init():2353] got version response upgrade_message: "wandb version 0.16.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-04-11 04:02:08,772 INFO MainThread:414 [wandb_init.py:init():805] starting run threads in backend +2024-04-11 04:02:24,756 INFO MainThread:414 [wandb_run.py:_console_start():2323] atexit reg +2024-04-11 04:02:24,757 INFO MainThread:414 [wandb_run.py:_redirect():2178] redirect: wrap_raw +2024-04-11 04:02:24,757 INFO MainThread:414 [wandb_run.py:_redirect():2243] Wrapping output streams. +2024-04-11 04:02:24,758 INFO MainThread:414 [wandb_run.py:_redirect():2268] Redirects installed. +2024-04-11 04:02:24,759 INFO MainThread:414 [wandb_init.py:init():848] run started, returning control to user process +2024-04-11 04:02:24,765 INFO MainThread:414 [wandb_run.py:_config_callback():1347] config_cb None None {'vocab_size': 250880, 'hidden_size': 1024, 'n_layer': 24, 'n_head': 16, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'use_cache': False, 'pretraining_tp': 1, 'apply_residual_connection_post_layernorm': False, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'bos_token_id': 1, 'eos_token_id': 2, 'slow_but_exact': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['BloomForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': 3, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'bigscience/bloomz-560m', 'transformers_version': '4.39.3', 'attention_softmax_in_fp32': True, 'bias_dropout_fusion': True, 'unk_token_id': 0, 'masked_softmax_fusion': True, 'model_type': 'bloom', 'n_inner': None, 'offset_alibi': 100, 'seq_length': 2048, 'skip_bias_add': True, 'skip_bias_add_qkv': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/kaggle/working/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 3e-05, 'weight_decay': 0.001, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 5, 'max_steps': 7000, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/kaggle/working/runs/Apr11_04-02-02_321ddc84c049', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 20, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 20, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/kaggle/working/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None} +2024-04-11 04:04:09,553 INFO MainThread:414 [jupyter.py:save_ipynb():373] not saving jupyter notebook +2024-04-11 04:04:09,553 INFO MainThread:414 [wandb_init.py:_pause_backend():438] pausing backend diff --git a/wandb/run-20240411_040208-txpa3x4k/run-txpa3x4k.wandb b/wandb/run-20240411_040208-txpa3x4k/run-txpa3x4k.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2c3d9528a7c038e24702a7c7bf849752e0f8faf9 Binary files /dev/null and b/wandb/run-20240411_040208-txpa3x4k/run-txpa3x4k.wandb differ diff --git a/wandb/run-20240411_040455-4dprer7c/files/conda-environment.yaml b/wandb/run-20240411_040455-4dprer7c/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/run-20240411_040455-4dprer7c/files/config.yaml b/wandb/run-20240411_040455-4dprer7c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fb57841e196eee3d1927fd57631d512970a3ec1 --- /dev/null +++ b/wandb/run-20240411_040455-4dprer7c/files/config.yaml @@ -0,0 +1,701 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.10.13 + cli_version: 0.16.5 + framework: huggingface + huggingface_version: 4.39.3 + is_jupyter_run: true + is_kaggle_kernel: true + start_time: 1712808295.0 + t: + 1: + - 1 + - 2 + - 3 + - 5 + - 11 + - 12 + - 49 + - 51 + - 53 + - 55 + - 71 + - 84 + - 98 + - 105 + 2: + - 1 + - 2 + - 3 + - 5 + - 11 + - 12 + - 49 + - 51 + - 53 + - 55 + - 71 + - 84 + - 98 + - 105 + 3: + - 7 + - 23 + 4: 3.10.13 + 5: 0.16.5 + 6: 4.39.3 + 8: + - 1 + - 2 + - 5 + 9: + 1: transformers_trainer + 13: linux-x86_64 + m: + - 1: train/global_step + 6: + - 3 + - 1: train/loss + 5: 1 + 6: + - 1 + - 1: train/grad_norm + 5: 1 + 6: + - 1 + - 1: train/learning_rate + 5: 1 + 6: + - 1 + - 1: train/epoch + 5: 1 + 6: + - 1 +vocab_size: + desc: null + value: 250880 +hidden_size: + desc: null + value: 1024 +n_layer: + desc: null + value: 24 +n_head: + desc: null + value: 16 +layer_norm_epsilon: + desc: null + value: 1.0e-05 +initializer_range: + desc: null + value: 0.02 +use_cache: + desc: null + value: false +pretraining_tp: + desc: null + value: 1 +apply_residual_connection_post_layernorm: + desc: null + value: false +hidden_dropout: + desc: null + value: 0.0 +attention_dropout: + desc: null + value: 0.0 +bos_token_id: + desc: null + value: 1 +eos_token_id: + desc: null + value: 2 +slow_but_exact: + desc: null + value: false +return_dict: + desc: null + value: true +output_hidden_states: + desc: null + value: false +output_attentions: + desc: null + value: false +torchscript: + desc: null + value: false +torch_dtype: + desc: null + value: null +use_bfloat16: + desc: null + value: false +tf_legacy_loss: + desc: null + value: false +pruned_heads: + desc: null + value: {} +tie_word_embeddings: + desc: null + value: true +chunk_size_feed_forward: + desc: null + value: 0 +is_encoder_decoder: + desc: null + value: false +is_decoder: + desc: null + value: false +cross_attention_hidden_size: + desc: null + value: null +add_cross_attention: + desc: null + value: false +tie_encoder_decoder: + desc: null + value: false +max_length: + desc: null + value: 20 +min_length: + desc: null + value: 0 +do_sample: + desc: null + value: false +early_stopping: + desc: null + value: false +num_beams: + desc: null + value: 1 +num_beam_groups: + desc: null + value: 1 +diversity_penalty: + desc: null + value: 0.0 +temperature: + desc: null + value: 1.0 +top_k: + desc: null + value: 50 +top_p: + desc: null + value: 1.0 +typical_p: + desc: null + value: 1.0 +repetition_penalty: + desc: null + value: 1.0 +length_penalty: + desc: null + value: 1.0 +no_repeat_ngram_size: + desc: null + value: 0 +encoder_no_repeat_ngram_size: + desc: null + value: 0 +bad_words_ids: + desc: null + value: null +num_return_sequences: + desc: null + value: 1 +output_scores: + desc: null + value: false +return_dict_in_generate: + desc: null + value: false +forced_bos_token_id: + desc: null + value: null +forced_eos_token_id: + desc: null + value: null +remove_invalid_values: + desc: null + value: false +exponential_decay_length_penalty: + desc: null + value: null +suppress_tokens: + desc: null + value: null +begin_suppress_tokens: + desc: null + value: null +architectures: + desc: null + value: + - BloomForCausalLM +finetuning_task: + desc: null + value: null +id2label: + desc: null + value: + '0': LABEL_0 + '1': LABEL_1 +label2id: + desc: null + value: + LABEL_0: 0 + LABEL_1: 1 +tokenizer_class: + desc: null + value: null +prefix: + desc: null + value: null +pad_token_id: + desc: null + value: 3 +sep_token_id: + desc: null + value: null +decoder_start_token_id: + desc: null + value: null +task_specific_params: + desc: null + value: null +problem_type: + desc: null + value: null +_name_or_path: + desc: null + value: bigscience/bloomz-560m +transformers_version: + desc: null + value: 4.39.3 +attention_softmax_in_fp32: + desc: null + value: true +bias_dropout_fusion: + desc: null + value: true +unk_token_id: + desc: null + value: 0 +masked_softmax_fusion: + desc: null + value: true +model_type: + desc: null + value: bloom +n_inner: + desc: null + value: null +offset_alibi: + desc: null + value: 100 +seq_length: + desc: null + value: 2048 +skip_bias_add: + desc: null + value: true +skip_bias_add_qkv: + desc: null + value: false +quantization_config: + desc: null + value: + quant_method: QuantizationMethod.BITS_AND_BYTES + _load_in_8bit: false + _load_in_4bit: true + llm_int8_threshold: 6.0 + llm_int8_skip_modules: null + llm_int8_enable_fp32_cpu_offload: false + llm_int8_has_fp16_weight: false + bnb_4bit_quant_type: nf4 + bnb_4bit_use_double_quant: false + bnb_4bit_compute_dtype: float16 + bnb_4bit_quant_storage: uint8 + load_in_4bit: true + load_in_8bit: false +output_dir: + desc: null + value: /kaggle/working/ +overwrite_output_dir: + desc: null + value: false +do_train: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +evaluation_strategy: + desc: null + value: 'no' +prediction_loss_only: + desc: null + value: false +per_device_train_batch_size: + desc: null + value: 8 +per_device_eval_batch_size: + desc: null + value: 8 +per_gpu_train_batch_size: + desc: null + value: null +per_gpu_eval_batch_size: + desc: null + value: null +gradient_accumulation_steps: + desc: null + value: 1 +eval_accumulation_steps: + desc: null + value: null +eval_delay: + desc: null + value: 0 +learning_rate: + desc: null + value: 3.0e-05 +weight_decay: + desc: null + value: 0.001 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.999 +adam_epsilon: + desc: null + value: 1.0e-08 +max_grad_norm: + desc: null + value: 0.3 +num_train_epochs: + desc: null + value: 5 +max_steps: + desc: null + value: 18000 +lr_scheduler_type: + desc: null + value: cosine +lr_scheduler_kwargs: + desc: null + value: {} +warmup_ratio: + desc: null + value: 0.03 +warmup_steps: + desc: null + value: 0 +log_level: + desc: null + value: passive +log_level_replica: + desc: null + value: warning +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: /kaggle/working/runs/Apr11_04-04-49_321ddc84c049 +logging_strategy: + desc: null + value: steps +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 20 +logging_nan_inf_filter: + desc: null + value: true +save_strategy: + desc: null + value: steps +save_steps: + desc: null + value: 20 +save_total_limit: + desc: null + value: 1 +save_safetensors: + desc: null + value: true +save_on_each_node: + desc: null + value: false +save_only_model: + desc: null + value: false +no_cuda: + desc: null + value: false +use_cpu: + desc: null + value: false +use_mps_device: + desc: null + value: false +seed: + desc: null + value: 42 +data_seed: + desc: null + value: null +jit_mode_eval: + desc: null + value: false +use_ipex: + desc: null + value: false +bf16: + desc: null + value: false +fp16: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +half_precision_backend: + desc: null + value: auto +bf16_full_eval: + desc: null + value: false +fp16_full_eval: + desc: null + value: false +tf32: + desc: null + value: null +local_rank: + desc: null + value: 0 +ddp_backend: + desc: null + value: null +tpu_num_cores: + desc: null + value: null +tpu_metrics_debug: + desc: null + value: false +debug: + desc: null + value: [] +dataloader_drop_last: + desc: null + value: false +eval_steps: + desc: null + value: null +dataloader_num_workers: + desc: null + value: 8 +dataloader_prefetch_factor: + desc: null + value: null +past_index: + desc: null + value: -1 +run_name: + desc: null + value: /kaggle/working/ +disable_tqdm: + desc: null + value: false +remove_unused_columns: + desc: null + value: true +label_names: + desc: null + value: null +load_best_model_at_end: + desc: null + value: false +metric_for_best_model: + desc: null + value: null +greater_is_better: + desc: null + value: null +ignore_data_skip: + desc: null + value: false +fsdp: + desc: null + value: [] +fsdp_min_num_params: + desc: null + value: 0 +fsdp_config: + desc: null + value: + min_num_params: 0 + xla: false + xla_fsdp_v2: false + xla_fsdp_grad_ckpt: false +fsdp_transformer_layer_cls_to_wrap: + desc: null + value: null +accelerator_config: + desc: null + value: + split_batches: false + dispatch_batches: null + even_batches: true + use_seedable_sampler: true +deepspeed: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +optim: + desc: null + value: paged_adamw_32bit +optim_args: + desc: null + value: null +adafactor: + desc: null + value: false +group_by_length: + desc: null + value: false +length_column_name: + desc: null + value: length +report_to: + desc: null + value: + - tensorboard + - wandb +ddp_find_unused_parameters: + desc: null + value: null +ddp_bucket_cap_mb: + desc: null + value: null +ddp_broadcast_buffers: + desc: null + value: null +dataloader_pin_memory: + desc: null + value: true +dataloader_persistent_workers: + desc: null + value: false +skip_memory_metrics: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +push_to_hub: + desc: null + value: false +resume_from_checkpoint: + desc: null + value: null +hub_model_id: + desc: null + value: null +hub_strategy: + desc: null + value: every_save +hub_token: + desc: null + value: +hub_private_repo: + desc: null + value: false +hub_always_push: + desc: null + value: false +gradient_checkpointing: + desc: null + value: true +gradient_checkpointing_kwargs: + desc: null + value: null +include_inputs_for_metrics: + desc: null + value: false +fp16_backend: + desc: null + value: auto +push_to_hub_model_id: + desc: null + value: null +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: +mp_parameters: + desc: null + value: '' +auto_find_batch_size: + desc: null + value: true +full_determinism: + desc: null + value: false +torchdynamo: + desc: null + value: null +ray_scope: + desc: null + value: last +ddp_timeout: + desc: null + value: 1800 +torch_compile: + desc: null + value: false +torch_compile_backend: + desc: null + value: null +torch_compile_mode: + desc: null + value: null +dispatch_batches: + desc: null + value: null +split_batches: + desc: null + value: null +include_tokens_per_second: + desc: null + value: false +include_num_input_tokens_seen: + desc: null + value: false +neftune_noise_alpha: + desc: null + value: null +optim_target_modules: + desc: null + value: null diff --git a/wandb/run-20240411_040455-4dprer7c/files/output.log b/wandb/run-20240411_040455-4dprer7c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f6629a1a9b12344d80f65bbf8043077753ea97ac --- /dev/null +++ b/wandb/run-20240411_040455-4dprer7c/files/output.log @@ -0,0 +1,12 @@ +/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:557: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary. + warnings.warn(_create_warning_msg( +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' \ No newline at end of file diff --git a/wandb/run-20240411_040455-4dprer7c/files/requirements.txt b/wandb/run-20240411_040455-4dprer7c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75d653a4d1ff18898bcd3ec2bda23c15dc7051d9 --- /dev/null +++ b/wandb/run-20240411_040455-4dprer7c/files/requirements.txt @@ -0,0 +1,864 @@ +Babel==2.14.0 +Boruta==0.3 +Brotli==1.0.9 +CVXcanon==0.1.2 +Cartopy==0.22.0 +Cython==3.0.8 +Deprecated==1.2.14 +Farama-Notifications==0.0.4 +Flask==3.0.2 +Geohash==1.0 +GitPython==3.1.41 +ImageHash==4.3.1 +Janome==0.5.0 +Jinja2==3.1.2 +LunarCalendar==0.0.9 +Mako==1.3.2 +Markdown==3.5.2 +MarkupSafe==2.1.3 +MarkupSafe==2.1.5 +Pillow==9.5.0 +PuLP==2.8.0 +PyArabic==0.6.15 +PyJWT==2.8.0 +PyMeeus==0.5.12 +PySocks==1.7.1 +PyUpSet==0.1.1.post7 +PyWavelets==1.5.0 +PyYAML==6.0.1 +Pygments==2.17.2 +Pympler==1.0.1 +QtPy==2.4.1 +Rtree==1.2.0 +SQLAlchemy==2.0.25 +SecretStorage==3.3.3 +Send2Trash==1.8.2 +Shapely==1.8.5.post1 +Shimmy==1.3.0 +SimpleITK==2.3.1 +TPOT==0.12.1 +Theano-PyMC==1.1.2 +Theano==1.0.5 +Wand==0.6.13 +Werkzeug==3.0.2 +absl-py==1.4.0 +accelerate==0.28.0 +access==1.1.9 +affine==2.4.0 +aiobotocore==2.12.2 +aiofiles==22.1.0 +aiohttp-cors==0.7.0 +aiohttp==3.9.1 +aioitertools==0.11.0 +aiorwlock==1.3.0 +aiosignal==1.3.1 +aiosqlite==0.19.0 +albumentations==1.4.0 +alembic==1.13.1 +altair==5.3.0 +annotated-types==0.6.0 +annoy==1.17.3 +anyio==4.2.0 +apache-beam==2.46.0 +aplus==0.11.0 +appdirs==1.4.4 +archspec==0.2.3 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +array-record==0.5.0 +arrow==1.3.0 +arviz==0.17.1 +astroid==3.1.0 +astropy-iers-data==0.2024.4.1.0.33.14 +astropy==6.0.1 +asttokens==2.4.1 +astunparse==1.6.3 +async-lru==2.0.4 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +autopep8==2.0.4 +backoff==2.2.1 +bayesian-optimization==1.4.3 +beatrix_jupyterlab==2023.128.151533 +beautifulsoup4==4.12.2 +bitsandbytes==0.43.0 +blake3==0.2.1 +bleach==6.1.0 +blessed==1.20.0 +blinker==1.7.0 +blis==0.7.10 +blosc2==2.6.0 +bokeh==3.3.4 +boltons==23.1.1 +boto3==1.26.100 +botocore==1.34.51 +bq_helper==0.4.1 +bqplot==0.12.43 +branca==0.7.1 +brewer2mpl==1.4.1 +brotlipy==0.7.0 +cached-property==1.5.2 +cachetools==4.2.4 +cachetools==5.3.2 +catalogue==2.0.10 +catalyst==22.4 +catboost==1.2.3 +category-encoders==2.6.3 +certifi==2024.2.2 +cesium==0.12.1 +cffi==1.16.0 +charset-normalizer==3.3.2 +chex==0.1.86 +cleverhans==4.0.0 +click-plugins==1.1.1 +click==8.1.7 +cligj==0.7.2 +cloud-tpu-client==0.10 +cloud-tpu-profiler==2.4.0 +cloudpathlib==0.16.0 +cloudpickle==2.2.1 +cloudpickle==3.0.0 +cmdstanpy==1.2.2 +colorama==0.4.6 +colorcet==3.1.0 +colorful==0.5.6 +colorlog==6.8.2 +colorlover==0.3.0 +comm==0.2.1 +conda-libmamba-solver==23.7.0 +conda-package-handling==2.2.0 +conda==23.7.4 +conda_package_streaming==0.9.0 +confection==0.1.4 +contextily==1.6.0 +contourpy==1.2.0 +convertdate==2.4.0 +crcmod==1.7 +cryptography==41.0.7 +cuda-python==12.4.0 +cudf==23.8.0 +cufflinks==0.17.3 +cuml==23.8.0 +cupy==13.0.0 +cycler==0.12.1 +cymem==2.0.8 +cytoolz==0.12.3 +daal4py==2024.2.0 +daal==2024.2.0 +dacite==1.8.1 +dask-cuda==23.8.0 +dask-cudf==23.8.0 +dask-expr==1.0.9 +dask==2024.4.0 +dataclasses-json==0.6.4 +dataproc_jupyter_plugin==0.1.66 +datasets==2.16.0 +datashader==0.16.0 +datatile==1.0.3 +db-dtypes==1.2.0 +deap==1.4.1 +debugpy==1.8.0 +decorator==5.1.1 +deepdiff==6.7.1 +defusedxml==0.7.1 +deprecation==2.1.0 +descartes==1.1.0 +dill==0.3.7 +dipy==1.9.0 +distlib==0.3.8 +distributed==2023.7.1 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docker==7.0.0 +docopt==0.6.2 +docstring-parser==0.15 +docstring-to-markdown==0.15 +docutils==0.20.1 +earthengine-api==0.1.395 +easydict==1.13 +easyocr==1.7.1 +ecos==2.0.13 +eli5==0.13.0 +emoji==2.11.0 +en-core-web-lg==3.7.1 +en-core-web-sm==3.7.1 +entrypoints==0.4 +ephem==4.1.5 +esda==2.5.1 +essentia==2.1b6.dev1110 +et-xmlfile==1.1.0 +etils==1.6.0 +exceptiongroup==1.2.0 +executing==2.0.1 +explainable-ai-sdk==1.3.3 +fastai==2.7.14 +fastapi==0.108.0 +fastavro==1.9.3 +fastcore==1.5.29 +fastdownload==0.0.7 +fasteners==0.19 +fastjsonschema==2.19.1 +fastprogress==1.0.3 +fastrlock==0.8.2 +fasttext==0.9.2 +feather-format==0.4.1 +featuretools==1.30.0 +filelock==3.13.1 +fiona==1.9.6 +fitter==1.7.0 +flake8==7.0.0 +flashtext==2.7 +flatbuffers==23.5.26 +flax==0.8.2 +folium==0.16.0 +fonttools==4.47.0 +fonttools==4.50.0 +fqdn==1.5.1 +frozendict==2.4.1 +frozenlist==1.4.1 +fsspec==2023.10.0 +fsspec==2024.3.1 +funcy==2.0 +fury==0.10.0 +future==1.0.0 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gatspy==0.3 +gcsfs==2024.2.0 +gensim==4.3.2 +geographiclib==2.0 +geojson==3.1.0 +geopandas==0.14.3 +geoplot==0.5.1 +geopy==2.4.1 +geoviews==1.11.1 +ggplot==0.11.5 +giddy==2.3.5 +gitdb==4.0.11 +google-ai-generativelanguage==0.4.0 +google-api-core==2.11.1 +google-api-core==2.18.0 +google-api-python-client==2.125.0 +google-apitools==0.5.31 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-auth==2.26.1 +google-cloud-aiplatform==0.6.0a1 +google-cloud-artifact-registry==1.10.0 +google-cloud-automl==1.0.1 +google-cloud-bigquery==2.34.4 +google-cloud-bigtable==1.7.3 +google-cloud-core==2.4.1 +google-cloud-datastore==2.19.0 +google-cloud-dlp==3.14.0 +google-cloud-jupyter-config==0.0.5 +google-cloud-language==2.13.3 +google-cloud-monitoring==2.18.0 +google-cloud-pubsub==2.19.0 +google-cloud-pubsublite==1.9.0 +google-cloud-recommendations-ai==0.7.1 +google-cloud-resource-manager==1.11.0 +google-cloud-spanner==3.40.1 +google-cloud-storage==1.44.0 +google-cloud-translate==3.12.1 +google-cloud-videointelligence==2.13.3 +google-cloud-vision==2.8.0 +google-crc32c==1.5.0 +google-generativeai==0.4.1 +google-pasta==0.2.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 +gplearn==0.4.2 +gpustat==1.0.0 +gpxpy==1.6.2 +graphviz==0.20.3 +greenlet==3.0.3 +grpc-google-iam-v1==0.12.7 +grpcio-status==1.48.1 +grpcio-status==1.48.2 +grpcio==1.51.1 +grpcio==1.60.0 +gviz-api==1.10.0 +gym-notices==0.0.8 +gym==0.26.2 +gymnasium==0.29.0 +h11==0.14.0 +h2o==3.46.0.1 +h5netcdf==1.3.0 +h5py==3.10.0 +haversine==2.8.1 +hdfs==2.7.3 +hep-ml==0.7.2 +hijri-converter==2.3.1 +hmmlearn==0.3.2 +holidays==0.24 +holoviews==1.18.3 +hpsklearn==0.1.0 +html5lib==1.1 +htmlmin==0.1.12 +httpcore==1.0.5 +httplib2==0.21.0 +httptools==0.6.1 +httpx==0.27.0 +huggingface-hub==0.22.2 +hunspell==0.5.5 +hydra-slayer==0.5.0 +hyperopt==0.2.7 +hypertools==0.8.0 +idna==3.6 +igraph==0.11.4 +imagecodecs==2024.1.1 +imageio==2.33.1 +imbalanced-learn==0.12.2 +imgaug==0.4.0 +importlib-metadata==6.11.0 +importlib-metadata==7.0.1 +importlib-resources==6.1.1 +inequality==1.0.1 +iniconfig==2.0.0 +ipydatawidgets==4.3.5 +ipykernel==6.28.0 +ipyleaflet==0.18.2 +ipympl==0.7.0 +ipython-genutils==0.2.0 +ipython-genutils==0.2.0 +ipython-sql==0.5.0 +ipython==8.20.0 +ipyvolume==0.6.3 +ipyvue==1.10.2 +ipyvuetify==1.9.3 +ipywebrtc==0.6.0 +ipywidgets==7.7.1 +isoduration==20.11.0 +isort==5.13.2 +isoweek==1.3.3 +itsdangerous==2.1.2 +jaraco.classes==3.3.0 +jax-jumpy==1.0.0 +jax==0.4.23 +jaxlib==0.4.23.dev20240116 +jedi==0.19.1 +jeepney==0.8.0 +jieba==0.42.1 +jmespath==1.0.1 +joblib==1.3.2 +json5==0.9.14 +jsonpatch==1.33 +jsonpointer==2.4 +jsonschema-specifications==2023.12.1 +jsonschema==4.20.0 +jupyter-console==6.6.3 +jupyter-events==0.9.0 +jupyter-http-over-ws==0.0.8 +jupyter-lsp==1.5.1 +jupyter-server-mathjax==0.2.6 +jupyter-ydoc==0.2.5 +jupyter_client==7.4.9 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +jupyter_server==2.13.0 +jupyter_server_fileid==0.9.1 +jupyter_server_proxy==4.1.0 +jupyter_server_terminals==0.5.1 +jupyter_server_ydoc==0.8.0 +jupyterlab-lsp==5.1.0 +jupyterlab-widgets==3.0.9 +jupyterlab==4.1.5 +jupyterlab_git==0.44.0 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.25.2 +jupytext==1.16.0 +kaggle-environments==1.14.3 +kaggle==1.6.8 +kagglehub==0.2.2 +keras-cv==0.8.2 +keras-nlp==0.8.2 +keras-tuner==1.4.6 +keras==3.1.1 +kernels-mixer==0.0.7 +keyring==24.3.0 +keyrings.google-artifactregistry-auth==1.1.2 +kfp-pipeline-spec==0.2.2 +kfp-server-api==2.0.5 +kfp==2.5.0 +kiwisolver==1.4.5 +kmapper==2.0.1 +kmodes==0.12.2 +korean-lunar-calendar==0.3.1 +kornia==0.7.2 +kornia_rs==0.1.3 +kt-legacy==1.0.5 +kubernetes==26.1.0 +langcodes==3.3.0 +langid==1.1.6 +lazy_loader==0.3 +learntools==0.3.4 +leven==1.0.4 +libclang==16.0.6 +libmambapy==1.5.0 +libpysal==4.9.2 +librosa==0.10.1 +lightgbm==4.2.0 +lightning-utilities==0.11.2 +lime==0.2.0.1 +line-profiler==4.1.2 +linkify-it-py==2.0.3 +llvmlite==0.41.1 +llvmlite==0.42.0 +lml==0.1.0 +locket==1.0.0 +loguru==0.7.2 +lxml==5.2.1 +lz4==4.3.3 +mamba==1.5.0 +mapclassify==2.6.1 +markdown-it-py==3.0.0 +marshmallow==3.21.1 +matplotlib-inline==0.1.6 +matplotlib-venn==0.11.10 +matplotlib==3.7.5 +matplotlib==3.8.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +memory-profiler==0.61.0 +menuinst==2.0.1 +mercantile==1.2.1 +mgwr==2.2.1 +missingno==0.5.2 +mistune==0.8.4 +mizani==0.11.1 +ml-dtypes==0.2.0 +mlcrate==0.2.0 +mlens==0.2.3 +mlxtend==0.23.1 +mne==1.6.1 +mnist==0.2.2 +momepy==0.7.0 +more-itertools==10.2.0 +mpld3==0.5.10 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multimethod==1.10 +multipledispatch==1.0.0 +multiprocess==0.70.15 +munkres==1.1.4 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +namex==0.0.7 +nb-conda-kernels==2.3.1 +nb_conda==2.2.1 +nbclassic==1.0.0 +nbclient==0.5.13 +nbconvert==6.4.5 +nbdime==3.2.0 +nbformat==5.9.2 +ndindex==1.8 +nest-asyncio==1.5.8 +networkx==3.2.1 +nibabel==5.2.1 +nilearn==0.10.3 +ninja==1.11.1.1 +nltk==3.2.4 +nose==1.3.7 +notebook==6.5.4 +notebook==6.5.6 +notebook_executor==0.2 +notebook_shim==0.2.3 +numba==0.58.1 +numba==0.59.1 +numexpr==2.10.0 +numpy==1.26.4 +nvidia-ml-py==11.495.46 +nvtx==0.2.10 +oauth2client==4.1.3 +oauthlib==3.2.2 +objsize==0.6.1 +odfpy==1.4.1 +olefile==0.47 +onnx==1.16.0 +opencensus-context==0.1.3 +opencensus==0.11.4 +opencv-contrib-python==4.9.0.80 +opencv-python-headless==4.9.0.80 +opencv-python==4.9.0.80 +openpyxl==3.1.2 +openslide-python==1.3.1 +opentelemetry-api==1.22.0 +opentelemetry-exporter-otlp-proto-common==1.22.0 +opentelemetry-exporter-otlp-proto-grpc==1.22.0 +opentelemetry-exporter-otlp-proto-http==1.22.0 +opentelemetry-exporter-otlp==1.22.0 +opentelemetry-proto==1.22.0 +opentelemetry-sdk==1.22.0 +opentelemetry-semantic-conventions==0.43b0 +opt-einsum==3.3.0 +optax==0.2.2 +optree==0.11.0 +optuna==3.6.1 +orbax-checkpoint==0.5.7 +ordered-set==4.1.0 +orjson==3.9.10 +ortools==9.4.1874 +osmnx==1.9.2 +overrides==7.4.0 +packaging==21.3 +pandas-datareader==0.10.0 +pandas-profiling==3.6.6 +pandas-summary==0.2.0 +pandas==2.1.4 +pandas==2.2.1 +pandasql==0.7.3 +pandocfilters==1.5.0 +panel==1.3.8 +papermill==2.5.0 +param==2.1.0 +parso==0.8.3 +partd==1.4.1 +path.py==12.5.0 +path==16.10.0 +pathos==0.3.2 +pathy==0.10.3 +patsy==0.5.6 +pdf2image==1.17.0 +peft==0.10.0 +pettingzoo==1.24.0 +pexpect==4.8.0 +pexpect==4.9.0 +phik==0.12.4 +pickleshare==0.7.5 +pillow==10.3.0 +pip==23.3.2 +pkgutil_resolve_name==1.3.10 +platformdirs==4.2.0 +plotly-express==0.4.1 +plotly==5.18.0 +plotnine==0.13.4 +pluggy==1.4.0 +pointpats==2.4.0 +polars==0.20.18 +polyglot==16.7.4 +pooch==1.8.1 +pox==0.3.4 +ppca==0.0.4 +ppft==1.7.6.8 +preprocessing==0.1.13 +preshed==3.0.9 +prettytable==3.9.0 +progressbar2==4.4.2 +prometheus-client==0.19.0 +promise==2.3 +prompt-toolkit==3.0.42 +prompt-toolkit==3.0.43 +prophet==1.1.1 +proto-plus==1.23.0 +protobuf==3.20.3 +protobuf==4.21.12 +psutil==5.9.3 +psutil==5.9.7 +ptyprocess==0.7.0 +pudb==2024.1 +pure-eval==0.2.2 +py-cpuinfo==9.0.0 +py-spy==0.3.14 +py4j==0.10.9.7 +pyLDAvis==3.4.1 +pyOpenSSL==23.3.0 +pyaml==23.12.0 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11==2.12.0 +pyclipper==1.3.0.post5 +pycodestyle==2.11.1 +pycosat==0.6.6 +pycparser==2.21 +pycryptodome==3.20.0 +pyct==0.5.0 +pycuda==2024.1 +pydantic==2.5.3 +pydantic==2.6.4 +pydantic_core==2.14.6 +pydantic_core==2.16.3 +pydegensac==0.1.2 +pydicom==2.4.4 +pydocstyle==6.3.0 +pydot==1.4.2 +pydub==0.25.1 +pyemd==1.0.0 +pyerfa==2.0.1.1 +pyexcel-io==0.6.6 +pyexcel-ods==0.6.0 +pyflakes==3.2.0 +pygltflib==1.16.2 +pykalman==0.9.7 +pylibraft==23.8.0 +pylint==3.1.0 +pymc3==3.11.4 +pymongo==3.13.0 +pynndescent==0.5.12 +pynvml==11.4.1 +pynvrtc==9.2 +pyparsing==3.1.1 +pyparsing==3.1.2 +pypdf==4.1.0 +pyproj==3.6.1 +pysal==24.1 +pyshp==2.3.1 +pytesseract==0.3.10 +pytest==8.1.1 +python-bidi==0.4.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.0 +python-json-logger==2.0.7 +python-louvain==0.16 +python-lsp-jsonrpc==1.1.2 +python-lsp-server==1.11.0 +python-slugify==8.0.4 +python-utils==3.8.2 +pythreejs==2.4.2 +pytoolconfig==1.3.1 +pytools==2024.1.1 +pytorch-ignite==0.5.0.post2 +pytorch-lightning==2.2.1 +pytz==2023.3.post1 +pytz==2024.1 +pyu2f==0.1.5 +pyviz_comms==3.0.2 +pyzmq==24.0.1 +pyzmq==25.1.2 +qgrid==1.3.1 +qtconsole==5.5.1 +quantecon==0.7.2 +qudida==0.0.4 +raft-dask==23.8.0 +rasterio==1.3.9 +rasterstats==0.19.0 +ray-cpp==2.9.0 +ray==2.9.0 +referencing==0.32.1 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==0.10.1 +requests==2.31.0 +retrying==1.3.3 +retrying==1.3.4 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rgf-python==3.12.0 +rich-click==1.7.4 +rich==13.7.0 +rich==13.7.1 +rmm==23.8.0 +rope==1.13.0 +rpds-py==0.16.2 +rsa==4.9 +ruamel-yaml-conda==0.15.100 +ruamel.yaml.clib==0.2.7 +ruamel.yaml==0.17.40 +s2sphere==0.2.5 +s3fs==2024.2.0 +s3transfer==0.6.2 +safetensors==0.4.2 +scattertext==0.1.19 +scikit-image==0.22.0 +scikit-learn-intelex==2024.2.0 +scikit-learn==1.2.2 +scikit-multilearn==0.2.0 +scikit-optimize==0.10.1 +scikit-plot==0.3.7 +scikit-surprise==1.1.3 +scipy==1.11.4 +scipy==1.12.0 +seaborn==0.12.2 +segment_anything==1.0 +segregation==2.5 +semver==3.0.2 +sentencepiece==0.2.0 +sentry-sdk==1.44.1 +setproctitle==1.3.3 +setuptools-git==1.2 +setuptools-scm==8.0.4 +setuptools==69.0.3 +shap==0.44.1 +shapely==2.0.3 +shellingham==1.5.4 +shtab==1.7.1 +simpervisor==1.0.0 +simplejson==3.19.2 +six==1.16.0 +sklearn-pandas==2.2.0 +slicer==0.0.7 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +spaghetti==1.7.5.post1 +spectral==0.23.1 +spglm==1.1.0 +sphinx-rtd-theme==0.2.4 +spint==1.0.7 +splot==1.1.5.post1 +spopt==0.6.0 +spreg==1.4.2 +spvcm==0.3.0 +sqlparse==0.4.4 +squarify==0.4.3 +srsly==2.4.8 +stable-baselines3==2.1.0 +stack-data==0.6.2 +stack-data==0.6.3 +stanio==0.5.0 +starlette==0.32.0.post1 +statsmodels==0.14.1 +stemming==1.0.1 +stop-words==2018.7.23 +stopit==1.1.2 +stumpy==1.12.0 +sympy==1.12 +tables==3.9.2 +tabulate==0.9.0 +tangled-up-in-unicode==0.2.0 +tbb==2021.12.0 +tblib==3.0.0 +tenacity==8.2.3 +tensorboard-data-server==0.7.2 +tensorboard-plugin-profile==2.15.0 +tensorboard==2.15.1 +tensorboardX==2.6.2.2 +tensorflow-cloud==0.1.16 +tensorflow-datasets==4.9.4 +tensorflow-decision-forests==1.8.1 +tensorflow-estimator==2.15.0 +tensorflow-hub==0.16.1 +tensorflow-io-gcs-filesystem==0.35.0 +tensorflow-io==0.35.0 +tensorflow-metadata==0.14.0 +tensorflow-probability==0.23.0 +tensorflow-serving-api==2.14.1 +tensorflow-text==2.15.0 +tensorflow-transform==0.14.0 +tensorflow==2.15.0 +tensorstore==0.1.56 +termcolor==2.4.0 +terminado==0.18.0 +testpath==0.6.0 +text-unidecode==1.3 +textblob==0.18.0.post0 +texttable==1.7.0 +tf_keras==2.15.1 +tfp-nightly==0.24.0.dev0 +thinc==8.2.2 +threadpoolctl==3.2.0 +tifffile==2023.12.9 +timm==0.9.16 +tinycss2==1.2.1 +tobler==0.11.2 +tokenizers==0.15.2 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.12.4 +toolz==0.12.1 +torch==2.1.2 +torchaudio==2.1.2 +torchdata==0.7.1 +torchinfo==1.8.0 +torchmetrics==1.3.2 +torchtext==0.16.2 +torchvision==0.16.2 +tornado==6.3.3 +tqdm==4.66.1 +traceml==1.0.8 +traitlets==5.9.0 +traittypes==0.2.1 +transformers==4.39.3 +treelite-runtime==3.2.0 +treelite==3.2.0 +trl==0.8.1 +truststore==0.8.0 +trx-python==0.2.9 +tsfresh==0.20.2 +typeguard==4.1.5 +typer==0.9.0 +typer==0.9.4 +types-python-dateutil==2.8.19.20240106 +typing-inspect==0.9.0 +typing-utils==0.1.0 +typing_extensions==4.9.0 +tyro==0.8.3 +tzdata==2023.4 +uc-micro-py==1.0.3 +ucx-py==0.33.0 +ujson==5.9.0 +umap-learn==0.5.5 +unicodedata2==15.1.0 +update-checker==0.18.0 +uri-template==1.3.0 +uritemplate==3.0.1 +urllib3==1.26.18 +urllib3==2.1.0 +urwid==2.6.10 +urwid_readline==0.14 +uvicorn==0.25.0 +uvloop==0.19.0 +vaex-astro==0.9.3 +vaex-core==4.17.1 +vaex-hdf5==0.14.1 +vaex-jupyter==0.8.2 +vaex-ml==0.18.3 +vaex-server==0.9.0 +vaex-viz==0.5.4 +vaex==4.17.0 +vec_noise==1.1.4 +vecstack==0.4.0 +virtualenv==20.21.0 +visions==0.7.5 +vowpalwabbit==9.9.0 +vtk==9.3.0 +wandb==0.16.5 +wasabi==1.1.2 +watchfiles==0.21.0 +wavio==0.0.8 +wcwidth==0.2.13 +weasel==0.3.4 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.7.0 +websockets==12.0 +wfdb==4.1.2 +whatthepatch==1.0.5 +wheel==0.42.0 +widgetsnbextension==3.6.6 +witwidget==1.8.1 +woodwork==0.29.0 +wordcloud==1.9.3 +wordsegment==1.3.1 +wrapt==1.14.1 +xarray-einstats==0.7.0 +xarray==2024.3.0 +xgboost==2.0.3 +xvfbwrapper==0.2.9 +xxhash==3.4.1 +xyzservices==2023.10.1 +y-py==0.6.2 +yapf==0.40.2 +yarl==1.9.3 +yarl==1.9.4 +ydata-profiling==4.6.4 +yellowbrick==1.5 +ypy-websocket==0.8.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.22.0 \ No newline at end of file diff --git a/wandb/run-20240411_040455-4dprer7c/files/wandb-metadata.json b/wandb/run-20240411_040455-4dprer7c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..833b4ea133afade17e801192f428def26a555306 --- /dev/null +++ b/wandb/run-20240411_040455-4dprer7c/files/wandb-metadata.json @@ -0,0 +1,66 @@ +{ + "os": "Linux-5.15.133+-x86_64-with-glibc2.31", + "python": "3.10.13", + "heartbeatAt": "2024-04-11T04:04:56.368183", + "startedAt": "2024-04-11T04:04:55.701503", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "kaggle.ipynb", + "codePathLocal": null, + "root": "/kaggle/working", + "host": "321ddc84c049", + "username": "root", + "executable": "/opt/conda/bin/python3.10", + "cpu_count": 2, + "cpu_count_logical": 4, + "cpu_freq": { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 8062.387607574463, + "used": 5566.962100982666 + } + }, + "gpu": "Tesla T4", + "gpu_count": 2, + "gpu_devices": [ + { + "name": "Tesla T4", + "memory_total": 16106127360 + }, + { + "name": "Tesla T4", + "memory_total": 16106127360 + } + ], + "memory": { + "total": 31.357559204101562 + } +} diff --git a/wandb/run-20240411_040455-4dprer7c/files/wandb-summary.json b/wandb/run-20240411_040455-4dprer7c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b56a13ea6dcb0f754e01d758b9298c4dbd7a756f --- /dev/null +++ b/wandb/run-20240411_040455-4dprer7c/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss": 30.0974, "train/grad_norm": 258.19573974609375, "train/learning_rate": 1.111111111111111e-06, "train/epoch": 0.0, "train/global_step": 20, "_timestamp": 1712808360.9619126, "_runtime": 65.25462675094604, "_step": 0, "_wandb": {"runtime": 106}} \ No newline at end of file diff --git a/wandb/run-20240411_040455-4dprer7c/logs/debug-internal.log b/wandb/run-20240411_040455-4dprer7c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4f95143ea6314bfea738d10fee3f889d3eb31b3a --- /dev/null +++ b/wandb/run-20240411_040455-4dprer7c/logs/debug-internal.log @@ -0,0 +1,254 @@ +2024-04-11 04:04:55,708 INFO StreamThr :781 [internal.py:wandb_internal():86] W&B internal server running at pid: 781, started at: 2024-04-11 04:04:55.707378 +2024-04-11 04:04:55,709 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status +2024-04-11 04:04:56,097 INFO WriterThread:781 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240411_040455-4dprer7c/run-4dprer7c.wandb +2024-04-11 04:04:56,097 DEBUG SenderThread:781 [sender.py:send():379] send: header +2024-04-11 04:04:56,100 DEBUG SenderThread:781 [sender.py:send():379] send: run +2024-04-11 04:04:56,242 INFO SenderThread:781 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files +2024-04-11 04:04:56,242 INFO SenderThread:781 [sender.py:_start_run_threads():1124] run started: 4dprer7c with start time 1712808295.707286 +2024-04-11 04:04:56,250 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: check_version +2024-04-11 04:04:56,250 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: check_version +2024-04-11 04:04:56,338 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: run_start +2024-04-11 04:04:56,349 DEBUG HandlerThread:781 [system_info.py:__init__():26] System info init +2024-04-11 04:04:56,349 DEBUG HandlerThread:781 [system_info.py:__init__():41] System info init done +2024-04-11 04:04:56,349 INFO HandlerThread:781 [system_monitor.py:start():194] Starting system monitor +2024-04-11 04:04:56,349 INFO SystemMonitor:781 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-11 04:04:56,349 INFO HandlerThread:781 [system_monitor.py:probe():214] Collecting system info +2024-04-11 04:04:56,350 INFO SystemMonitor:781 [interfaces.py:start():190] Started cpu monitoring +2024-04-11 04:04:56,351 INFO SystemMonitor:781 [interfaces.py:start():190] Started disk monitoring +2024-04-11 04:04:56,352 INFO SystemMonitor:781 [interfaces.py:start():190] Started gpu monitoring +2024-04-11 04:04:56,352 INFO SystemMonitor:781 [interfaces.py:start():190] Started memory monitoring +2024-04-11 04:04:56,353 INFO SystemMonitor:781 [interfaces.py:start():190] Started network monitoring +2024-04-11 04:04:56,368 DEBUG HandlerThread:781 [system_info.py:probe():150] Probing system +2024-04-11 04:04:56,370 DEBUG HandlerThread:781 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-11 04:04:56,370 DEBUG HandlerThread:781 [system_info.py:probe():198] Probing system done +2024-04-11 04:04:56,370 DEBUG HandlerThread:781 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-11T04:04:56.368183', 'startedAt': '2024-04-11T04:04:55.701503', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '321ddc84c049', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.19, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5566.962100982666}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-04-11 04:04:56,370 INFO HandlerThread:781 [system_monitor.py:probe():224] Finished collecting system info +2024-04-11 04:04:56,370 INFO HandlerThread:781 [system_monitor.py:probe():227] Publishing system info +2024-04-11 04:04:56,370 DEBUG HandlerThread:781 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-11 04:04:57,244 INFO Thread-12 :781 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/conda-environment.yaml +2024-04-11 04:05:11,386 ERROR HandlerThread:781 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +Traceback (most recent call last): + File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda + subprocess.call( + File "/opt/conda/lib/python3.10/subprocess.py", line 347, in call + return p.wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1209, in wait + return self._wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait + raise TimeoutExpired(self.args, timeout) +subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-04-11 04:05:11,386 DEBUG HandlerThread:781 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-11 04:05:11,387 INFO HandlerThread:781 [system_monitor.py:probe():229] Finished publishing system info +2024-04-11 04:05:11,392 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:11,393 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:05:11,393 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:11,393 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:05:11,393 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:11,393 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:05:11,393 DEBUG SenderThread:781 [sender.py:send():379] send: files +2024-04-11 04:05:11,394 INFO SenderThread:781 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-11 04:05:11,591 INFO wandb-upload_0:781 [upload_job.py:push():131] Uploaded file /tmp/tmp48h4betowandb/odf1mr1f-wandb-metadata.json +2024-04-11 04:05:12,247 INFO Thread-12 :781 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/wandb-metadata.json +2024-04-11 04:05:12,360 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: python_packages +2024-04-11 04:05:12,360 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: python_packages +2024-04-11 04:05:12,363 DEBUG SenderThread:781 [sender.py:send():379] send: telemetry +2024-04-11 04:05:12,373 DEBUG SenderThread:781 [sender.py:send():379] send: config +2024-04-11 04:05:12,375 DEBUG SenderThread:781 [sender.py:send():379] send: metric +2024-04-11 04:05:12,376 DEBUG SenderThread:781 [sender.py:send():379] send: telemetry +2024-04-11 04:05:12,377 DEBUG SenderThread:781 [sender.py:send():379] send: metric +2024-04-11 04:05:12,377 WARNING SenderThread:781 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-11 04:05:12,377 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:05:12,377 DEBUG SenderThread:781 [sender.py:send():379] send: telemetry +2024-04-11 04:05:12,378 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:05:12,379 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:05:13,248 INFO Thread-12 :781 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log +2024-04-11 04:05:13,248 INFO Thread-12 :781 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/requirements.txt +2024-04-11 04:05:15,249 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log +2024-04-11 04:05:16,873 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:17,250 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log +2024-04-11 04:05:21,874 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:26,880 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:27,254 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/config.yaml +2024-04-11 04:05:28,163 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:05:28,165 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:05:28,166 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:05:32,273 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:37,274 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:42,275 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:42,872 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:05:42,872 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:05:42,873 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:05:47,920 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:52,920 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:05:56,353 DEBUG SystemMonitor:781 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-11 04:05:56,355 DEBUG SenderThread:781 [sender.py:send():379] send: stats +2024-04-11 04:05:57,869 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:05:57,870 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:05:57,903 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:05:58,904 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:00,962 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:06:00,965 DEBUG SenderThread:781 [sender.py:send():379] send: metric +2024-04-11 04:06:00,965 DEBUG SenderThread:781 [sender.py:send():379] send: metric +2024-04-11 04:06:00,965 DEBUG SenderThread:781 [sender.py:send():379] send: metric +2024-04-11 04:06:00,965 DEBUG SenderThread:781 [sender.py:send():379] send: metric +2024-04-11 04:06:00,966 DEBUG SenderThread:781 [sender.py:send():379] send: history +2024-04-11 04:06:00,966 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:06:00,966 INFO SenderThread:781 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:06:01,267 INFO Thread-12 :781 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/wandb-summary.json +2024-04-11 04:06:03,268 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log +2024-04-11 04:06:04,004 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:07,270 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log +2024-04-11 04:06:09,172 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:12,869 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:06:12,870 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:06:12,873 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:06:14,934 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:19,934 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:24,936 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:26,356 DEBUG SenderThread:781 [sender.py:send():379] send: stats +2024-04-11 04:06:27,870 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:06:27,870 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:06:27,910 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:06:29,974 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:30,281 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/config.yaml +2024-04-11 04:06:35,067 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:40,068 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:42,978 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: pause +2024-04-11 04:06:42,978 INFO HandlerThread:781 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-11 04:06:42,978 INFO HandlerThread:781 [system_monitor.py:finish():203] Stopping system monitor +2024-04-11 04:06:42,978 INFO HandlerThread:781 [interfaces.py:finish():202] Joined cpu monitor +2024-04-11 04:06:42,979 INFO HandlerThread:781 [interfaces.py:finish():202] Joined disk monitor +2024-04-11 04:06:42,979 DEBUG SystemMonitor:781 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-11 04:06:42,979 DEBUG SystemMonitor:781 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-11 04:06:42,990 INFO HandlerThread:781 [interfaces.py:finish():202] Joined gpu monitor +2024-04-11 04:06:42,990 INFO HandlerThread:781 [interfaces.py:finish():202] Joined memory monitor +2024-04-11 04:06:42,990 INFO HandlerThread:781 [interfaces.py:finish():202] Joined network monitor +2024-04-11 04:06:42,991 DEBUG SenderThread:781 [sender.py:send():379] send: stats +2024-04-11 04:06:43,019 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:06:43,019 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:06:43,019 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:06:46,066 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:49,976 DEBUG SenderThread:781 [sender.py:send():379] send: exit +2024-04-11 04:06:49,976 INFO SenderThread:781 [sender.py:send_exit():586] handling exit code: 0 +2024-04-11 04:06:49,976 INFO SenderThread:781 [sender.py:send_exit():588] handling runtime: 106 +2024-04-11 04:06:49,977 INFO SenderThread:781 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:06:49,977 INFO SenderThread:781 [sender.py:send_exit():594] send defer +2024-04-11 04:06:49,977 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,977 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 0 +2024-04-11 04:06:49,977 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,978 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-04-11 04:06:49,978 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 1 +2024-04-11 04:06:49,978 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,978 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 1 +2024-04-11 04:06:49,978 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,978 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-04-11 04:06:49,978 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 2 +2024-04-11 04:06:49,978 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,978 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 2 +2024-04-11 04:06:49,978 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,978 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-04-11 04:06:49,978 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 3 +2024-04-11 04:06:49,979 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,979 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 3 +2024-04-11 04:06:49,979 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,979 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-04-11 04:06:49,979 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 4 +2024-04-11 04:06:49,979 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,979 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 4 +2024-04-11 04:06:49,979 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,979 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-04-11 04:06:49,979 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 5 +2024-04-11 04:06:49,979 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,980 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 5 +2024-04-11 04:06:49,980 DEBUG SenderThread:781 [sender.py:send():379] send: summary +2024-04-11 04:06:49,980 INFO SenderThread:781 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:06:49,980 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,981 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-04-11 04:06:49,981 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 6 +2024-04-11 04:06:49,981 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,981 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 6 +2024-04-11 04:06:49,981 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,981 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-04-11 04:06:49,981 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 7 +2024-04-11 04:06:49,981 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:06:49,981 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:49,981 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 7 +2024-04-11 04:06:49,982 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:49,982 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-04-11 04:06:50,288 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/wandb-summary.json +2024-04-11 04:06:50,499 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 8 +2024-04-11 04:06:50,499 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:50,499 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 8 +2024-04-11 04:06:50,499 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:50,499 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-04-11 04:06:50,500 INFO SenderThread:781 [job_builder.py:build():318] Attempting to build job artifact +2024-04-11 04:06:50,501 INFO SenderThread:781 [job_builder.py:_get_source_type():466] no source found +2024-04-11 04:06:50,502 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 9 +2024-04-11 04:06:50,502 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:50,502 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 9 +2024-04-11 04:06:50,502 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:50,502 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-04-11 04:06:50,502 INFO SenderThread:781 [dir_watcher.py:finish():358] shutting down directory watcher +2024-04-11 04:06:50,976 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:06:51,289 INFO Thread-12 :781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log +2024-04-11 04:06:51,289 INFO SenderThread:781 [dir_watcher.py:finish():388] scan: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files +2024-04-11 04:06:51,290 INFO SenderThread:781 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/config.yaml config.yaml +2024-04-11 04:06:51,290 INFO SenderThread:781 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/requirements.txt requirements.txt +2024-04-11 04:06:51,294 INFO SenderThread:781 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/conda-environment.yaml conda-environment.yaml +2024-04-11 04:06:51,298 INFO SenderThread:781 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/wandb-summary.json wandb-summary.json +2024-04-11 04:06:51,299 INFO SenderThread:781 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/wandb-metadata.json wandb-metadata.json +2024-04-11 04:06:51,303 INFO SenderThread:781 [dir_watcher.py:finish():402] scan save: /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log output.log +2024-04-11 04:06:51,304 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 10 +2024-04-11 04:06:51,305 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:06:51,308 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:51,313 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 10 +2024-04-11 04:06:51,313 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:51,313 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-04-11 04:06:51,313 INFO SenderThread:781 [file_pusher.py:finish():172] shutting down file pusher +2024-04-11 04:06:51,460 INFO wandb-upload_0:781 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/config.yaml +2024-04-11 04:06:51,532 INFO wandb-upload_1:781 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/requirements.txt +2024-04-11 04:06:51,536 INFO wandb-upload_3:781 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/output.log +2024-04-11 04:06:51,604 INFO wandb-upload_2:781 [upload_job.py:push():131] Uploaded file /kaggle/working/wandb/run-20240411_040455-4dprer7c/files/wandb-summary.json +2024-04-11 04:06:51,804 INFO Thread-11 (_thread_body):781 [sender.py:transition_state():614] send defer: 11 +2024-04-11 04:06:51,805 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:51,805 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 11 +2024-04-11 04:06:51,805 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:51,805 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-04-11 04:06:51,805 INFO SenderThread:781 [file_pusher.py:join():178] waiting for file pusher +2024-04-11 04:06:51,806 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 12 +2024-04-11 04:06:51,806 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:51,806 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 12 +2024-04-11 04:06:51,806 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:51,806 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-04-11 04:06:51,806 INFO SenderThread:781 [file_stream.py:finish():614] file stream finish called +2024-04-11 04:06:51,868 INFO SenderThread:781 [file_stream.py:finish():618] file stream finish is done +2024-04-11 04:06:51,869 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 13 +2024-04-11 04:06:51,869 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:51,869 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 13 +2024-04-11 04:06:51,869 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:51,869 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-04-11 04:06:51,869 INFO SenderThread:781 [sender.py:transition_state():614] send defer: 14 +2024-04-11 04:06:51,869 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: defer +2024-04-11 04:06:51,869 INFO HandlerThread:781 [handler.py:handle_request_defer():172] handle defer: 14 +2024-04-11 04:06:51,870 DEBUG SenderThread:781 [sender.py:send():379] send: final +2024-04-11 04:06:51,870 DEBUG SenderThread:781 [sender.py:send():379] send: footer +2024-04-11 04:06:51,870 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: defer +2024-04-11 04:06:51,870 INFO SenderThread:781 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-04-11 04:06:51,871 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:06:51,872 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:06:51,872 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: poll_exit +2024-04-11 04:06:51,873 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: poll_exit +2024-04-11 04:06:51,873 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: server_info +2024-04-11 04:06:51,873 DEBUG SenderThread:781 [sender.py:send_request():406] send_request: server_info +2024-04-11 04:06:51,876 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: get_summary +2024-04-11 04:06:51,876 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: sampled_history +2024-04-11 04:06:51,877 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:06:51,888 INFO MainThread:781 [wandb_run.py:_footer_history_summary_info():3920] rendering history +2024-04-11 04:06:51,888 INFO MainThread:781 [wandb_run.py:_footer_history_summary_info():3952] rendering summary +2024-04-11 04:06:51,889 INFO MainThread:781 [wandb_run.py:_footer_sync_info():3879] logging synced files +2024-04-11 04:06:51,889 DEBUG HandlerThread:781 [handler.py:handle_request():146] handle_request: shutdown +2024-04-11 04:06:51,889 INFO HandlerThread:781 [handler.py:finish():866] shutting down handler +2024-04-11 04:06:52,873 INFO WriterThread:781 [datastore.py:close():296] close: /kaggle/working/wandb/run-20240411_040455-4dprer7c/run-4dprer7c.wandb +2024-04-11 04:06:52,888 INFO SenderThread:781 [sender.py:finish():1546] shutting down sender +2024-04-11 04:06:52,888 INFO SenderThread:781 [file_pusher.py:finish():172] shutting down file pusher +2024-04-11 04:06:52,888 INFO SenderThread:781 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240411_040455-4dprer7c/logs/debug.log b/wandb/run-20240411_040455-4dprer7c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..815bb91b9fd83d6d36d2ba871dc2d3e4e2c56571 --- /dev/null +++ b/wandb/run-20240411_040455-4dprer7c/logs/debug.log @@ -0,0 +1,33 @@ +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Current SDK version is 0.16.5 +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Configure stats pid to 740 +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Loading settings from /kaggle/working/wandb/settings +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': ''} +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_init.py:_log_setup():527] Logging user logs to /kaggle/working/wandb/run-20240411_040455-4dprer7c/logs/debug.log +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_init.py:_log_setup():528] Logging internal logs to /kaggle/working/wandb/run-20240411_040455-4dprer7c/logs/debug-internal.log +2024-04-11 04:04:55,703 INFO MainThread:740 [wandb_init.py:_jupyter_setup():473] configuring jupyter hooks +2024-04-11 04:04:55,704 INFO MainThread:740 [wandb_init.py:init():567] calling init triggers +2024-04-11 04:04:55,704 INFO MainThread:740 [wandb_init.py:init():574] wandb.init called with sweep_config: {} +config: {} +2024-04-11 04:04:55,704 INFO MainThread:740 [wandb_init.py:init():617] starting backend +2024-04-11 04:04:55,704 INFO MainThread:740 [wandb_init.py:init():621] setting up manager +2024-04-11 04:04:55,706 INFO MainThread:740 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-04-11 04:04:55,707 INFO MainThread:740 [wandb_init.py:init():629] backend started and connected +2024-04-11 04:04:55,718 INFO MainThread:740 [wandb_run.py:_label_probe_notebook():1299] probe notebook +2024-04-11 04:04:56,096 INFO MainThread:740 [wandb_init.py:init():721] updated telemetry +2024-04-11 04:04:56,099 INFO MainThread:740 [wandb_init.py:init():754] communicating run to backend with 90.0 second timeout +2024-04-11 04:04:56,249 INFO MainThread:740 [wandb_run.py:_on_init():2344] communicating current version +2024-04-11 04:04:56,332 INFO MainThread:740 [wandb_run.py:_on_init():2353] got version response upgrade_message: "wandb version 0.16.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-04-11 04:04:56,332 INFO MainThread:740 [wandb_init.py:init():805] starting run threads in backend +2024-04-11 04:05:12,361 INFO MainThread:740 [wandb_run.py:_console_start():2323] atexit reg +2024-04-11 04:05:12,361 INFO MainThread:740 [wandb_run.py:_redirect():2178] redirect: wrap_raw +2024-04-11 04:05:12,362 INFO MainThread:740 [wandb_run.py:_redirect():2243] Wrapping output streams. +2024-04-11 04:05:12,362 INFO MainThread:740 [wandb_run.py:_redirect():2268] Redirects installed. +2024-04-11 04:05:12,363 INFO MainThread:740 [wandb_init.py:init():848] run started, returning control to user process +2024-04-11 04:05:12,368 INFO MainThread:740 [wandb_run.py:_config_callback():1347] config_cb None None {'vocab_size': 250880, 'hidden_size': 1024, 'n_layer': 24, 'n_head': 16, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'use_cache': False, 'pretraining_tp': 1, 'apply_residual_connection_post_layernorm': False, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'bos_token_id': 1, 'eos_token_id': 2, 'slow_but_exact': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['BloomForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': 3, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'bigscience/bloomz-560m', 'transformers_version': '4.39.3', 'attention_softmax_in_fp32': True, 'bias_dropout_fusion': True, 'unk_token_id': 0, 'masked_softmax_fusion': True, 'model_type': 'bloom', 'n_inner': None, 'offset_alibi': 100, 'seq_length': 2048, 'skip_bias_add': True, 'skip_bias_add_qkv': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/kaggle/working/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 3e-05, 'weight_decay': 0.001, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 5, 'max_steps': 18000, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/kaggle/working/runs/Apr11_04-04-49_321ddc84c049', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 20, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 20, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/kaggle/working/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None} +2024-04-11 04:06:42,977 INFO MainThread:740 [jupyter.py:save_ipynb():373] not saving jupyter notebook +2024-04-11 04:06:42,977 INFO MainThread:740 [wandb_init.py:_pause_backend():438] pausing backend diff --git a/wandb/run-20240411_040455-4dprer7c/run-4dprer7c.wandb b/wandb/run-20240411_040455-4dprer7c/run-4dprer7c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2d0c97c9b72ae19c373c0fa557d0aa239ac1d6ec Binary files /dev/null and b/wandb/run-20240411_040455-4dprer7c/run-4dprer7c.wandb differ diff --git a/wandb/run-20240411_040722-rimyh9a1/files/conda-environment.yaml b/wandb/run-20240411_040722-rimyh9a1/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/run-20240411_040722-rimyh9a1/files/config.yaml b/wandb/run-20240411_040722-rimyh9a1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..957295db1772c1e2383a6650cf2f79b108b2d8b4 --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/files/config.yaml @@ -0,0 +1,702 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.10.13 + cli_version: 0.16.5 + framework: huggingface + huggingface_version: 4.39.3 + is_jupyter_run: true + is_kaggle_kernel: true + start_time: 1712808442.0 + t: + 1: + - 1 + - 2 + - 3 + - 5 + - 11 + - 12 + - 49 + - 51 + - 53 + - 55 + - 71 + - 84 + - 98 + - 105 + 2: + - 1 + - 2 + - 3 + - 5 + - 11 + - 12 + - 49 + - 51 + - 53 + - 55 + - 71 + - 84 + - 98 + - 105 + 3: + - 7 + - 23 + - 62 + 4: 3.10.13 + 5: 0.16.5 + 6: 4.39.3 + 8: + - 1 + - 2 + - 5 + 9: + 1: transformers_trainer + 13: linux-x86_64 + m: + - 1: train/global_step + 6: + - 3 + - 1: train/loss + 5: 1 + 6: + - 1 + - 1: train/grad_norm + 5: 1 + 6: + - 1 + - 1: train/learning_rate + 5: 1 + 6: + - 1 + - 1: train/epoch + 5: 1 + 6: + - 1 +vocab_size: + desc: null + value: 250880 +hidden_size: + desc: null + value: 1024 +n_layer: + desc: null + value: 24 +n_head: + desc: null + value: 16 +layer_norm_epsilon: + desc: null + value: 1.0e-05 +initializer_range: + desc: null + value: 0.02 +use_cache: + desc: null + value: false +pretraining_tp: + desc: null + value: 1 +apply_residual_connection_post_layernorm: + desc: null + value: false +hidden_dropout: + desc: null + value: 0.0 +attention_dropout: + desc: null + value: 0.0 +bos_token_id: + desc: null + value: 1 +eos_token_id: + desc: null + value: 2 +slow_but_exact: + desc: null + value: false +return_dict: + desc: null + value: true +output_hidden_states: + desc: null + value: false +output_attentions: + desc: null + value: false +torchscript: + desc: null + value: false +torch_dtype: + desc: null + value: null +use_bfloat16: + desc: null + value: false +tf_legacy_loss: + desc: null + value: false +pruned_heads: + desc: null + value: {} +tie_word_embeddings: + desc: null + value: true +chunk_size_feed_forward: + desc: null + value: 0 +is_encoder_decoder: + desc: null + value: false +is_decoder: + desc: null + value: false +cross_attention_hidden_size: + desc: null + value: null +add_cross_attention: + desc: null + value: false +tie_encoder_decoder: + desc: null + value: false +max_length: + desc: null + value: 20 +min_length: + desc: null + value: 0 +do_sample: + desc: null + value: false +early_stopping: + desc: null + value: false +num_beams: + desc: null + value: 1 +num_beam_groups: + desc: null + value: 1 +diversity_penalty: + desc: null + value: 0.0 +temperature: + desc: null + value: 1.0 +top_k: + desc: null + value: 50 +top_p: + desc: null + value: 1.0 +typical_p: + desc: null + value: 1.0 +repetition_penalty: + desc: null + value: 1.0 +length_penalty: + desc: null + value: 1.0 +no_repeat_ngram_size: + desc: null + value: 0 +encoder_no_repeat_ngram_size: + desc: null + value: 0 +bad_words_ids: + desc: null + value: null +num_return_sequences: + desc: null + value: 1 +output_scores: + desc: null + value: false +return_dict_in_generate: + desc: null + value: false +forced_bos_token_id: + desc: null + value: null +forced_eos_token_id: + desc: null + value: null +remove_invalid_values: + desc: null + value: false +exponential_decay_length_penalty: + desc: null + value: null +suppress_tokens: + desc: null + value: null +begin_suppress_tokens: + desc: null + value: null +architectures: + desc: null + value: + - BloomForCausalLM +finetuning_task: + desc: null + value: null +id2label: + desc: null + value: + '0': LABEL_0 + '1': LABEL_1 +label2id: + desc: null + value: + LABEL_0: 0 + LABEL_1: 1 +tokenizer_class: + desc: null + value: null +prefix: + desc: null + value: null +pad_token_id: + desc: null + value: 3 +sep_token_id: + desc: null + value: null +decoder_start_token_id: + desc: null + value: null +task_specific_params: + desc: null + value: null +problem_type: + desc: null + value: null +_name_or_path: + desc: null + value: bigscience/bloomz-560m +transformers_version: + desc: null + value: 4.39.3 +attention_softmax_in_fp32: + desc: null + value: true +bias_dropout_fusion: + desc: null + value: true +unk_token_id: + desc: null + value: 0 +masked_softmax_fusion: + desc: null + value: true +model_type: + desc: null + value: bloom +n_inner: + desc: null + value: null +offset_alibi: + desc: null + value: 100 +seq_length: + desc: null + value: 2048 +skip_bias_add: + desc: null + value: true +skip_bias_add_qkv: + desc: null + value: false +quantization_config: + desc: null + value: + quant_method: QuantizationMethod.BITS_AND_BYTES + _load_in_8bit: false + _load_in_4bit: true + llm_int8_threshold: 6.0 + llm_int8_skip_modules: null + llm_int8_enable_fp32_cpu_offload: false + llm_int8_has_fp16_weight: false + bnb_4bit_quant_type: nf4 + bnb_4bit_use_double_quant: false + bnb_4bit_compute_dtype: float16 + bnb_4bit_quant_storage: uint8 + load_in_4bit: true + load_in_8bit: false +output_dir: + desc: null + value: /kaggle/working/ +overwrite_output_dir: + desc: null + value: false +do_train: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +evaluation_strategy: + desc: null + value: 'no' +prediction_loss_only: + desc: null + value: false +per_device_train_batch_size: + desc: null + value: 8 +per_device_eval_batch_size: + desc: null + value: 8 +per_gpu_train_batch_size: + desc: null + value: null +per_gpu_eval_batch_size: + desc: null + value: null +gradient_accumulation_steps: + desc: null + value: 1 +eval_accumulation_steps: + desc: null + value: null +eval_delay: + desc: null + value: 0 +learning_rate: + desc: null + value: 3.0e-05 +weight_decay: + desc: null + value: 0.001 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.999 +adam_epsilon: + desc: null + value: 1.0e-08 +max_grad_norm: + desc: null + value: 0.3 +num_train_epochs: + desc: null + value: 5 +max_steps: + desc: null + value: 20000 +lr_scheduler_type: + desc: null + value: cosine +lr_scheduler_kwargs: + desc: null + value: {} +warmup_ratio: + desc: null + value: 0.03 +warmup_steps: + desc: null + value: 0 +log_level: + desc: null + value: passive +log_level_replica: + desc: null + value: warning +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: /kaggle/working/runs/Apr11_04-07-17_321ddc84c049 +logging_strategy: + desc: null + value: steps +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 20 +logging_nan_inf_filter: + desc: null + value: true +save_strategy: + desc: null + value: steps +save_steps: + desc: null + value: 20 +save_total_limit: + desc: null + value: 1 +save_safetensors: + desc: null + value: true +save_on_each_node: + desc: null + value: false +save_only_model: + desc: null + value: false +no_cuda: + desc: null + value: false +use_cpu: + desc: null + value: false +use_mps_device: + desc: null + value: false +seed: + desc: null + value: 42 +data_seed: + desc: null + value: null +jit_mode_eval: + desc: null + value: false +use_ipex: + desc: null + value: false +bf16: + desc: null + value: false +fp16: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +half_precision_backend: + desc: null + value: auto +bf16_full_eval: + desc: null + value: false +fp16_full_eval: + desc: null + value: false +tf32: + desc: null + value: null +local_rank: + desc: null + value: 0 +ddp_backend: + desc: null + value: null +tpu_num_cores: + desc: null + value: null +tpu_metrics_debug: + desc: null + value: false +debug: + desc: null + value: [] +dataloader_drop_last: + desc: null + value: false +eval_steps: + desc: null + value: null +dataloader_num_workers: + desc: null + value: 8 +dataloader_prefetch_factor: + desc: null + value: null +past_index: + desc: null + value: -1 +run_name: + desc: null + value: /kaggle/working/ +disable_tqdm: + desc: null + value: false +remove_unused_columns: + desc: null + value: true +label_names: + desc: null + value: null +load_best_model_at_end: + desc: null + value: false +metric_for_best_model: + desc: null + value: null +greater_is_better: + desc: null + value: null +ignore_data_skip: + desc: null + value: false +fsdp: + desc: null + value: [] +fsdp_min_num_params: + desc: null + value: 0 +fsdp_config: + desc: null + value: + min_num_params: 0 + xla: false + xla_fsdp_v2: false + xla_fsdp_grad_ckpt: false +fsdp_transformer_layer_cls_to_wrap: + desc: null + value: null +accelerator_config: + desc: null + value: + split_batches: false + dispatch_batches: null + even_batches: true + use_seedable_sampler: true +deepspeed: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +optim: + desc: null + value: paged_adamw_32bit +optim_args: + desc: null + value: null +adafactor: + desc: null + value: false +group_by_length: + desc: null + value: false +length_column_name: + desc: null + value: length +report_to: + desc: null + value: + - tensorboard + - wandb +ddp_find_unused_parameters: + desc: null + value: null +ddp_bucket_cap_mb: + desc: null + value: null +ddp_broadcast_buffers: + desc: null + value: null +dataloader_pin_memory: + desc: null + value: true +dataloader_persistent_workers: + desc: null + value: false +skip_memory_metrics: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +push_to_hub: + desc: null + value: false +resume_from_checkpoint: + desc: null + value: null +hub_model_id: + desc: null + value: null +hub_strategy: + desc: null + value: every_save +hub_token: + desc: null + value: +hub_private_repo: + desc: null + value: false +hub_always_push: + desc: null + value: false +gradient_checkpointing: + desc: null + value: true +gradient_checkpointing_kwargs: + desc: null + value: null +include_inputs_for_metrics: + desc: null + value: false +fp16_backend: + desc: null + value: auto +push_to_hub_model_id: + desc: null + value: null +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: +mp_parameters: + desc: null + value: '' +auto_find_batch_size: + desc: null + value: true +full_determinism: + desc: null + value: false +torchdynamo: + desc: null + value: null +ray_scope: + desc: null + value: last +ddp_timeout: + desc: null + value: 1800 +torch_compile: + desc: null + value: false +torch_compile_backend: + desc: null + value: null +torch_compile_mode: + desc: null + value: null +dispatch_batches: + desc: null + value: null +split_batches: + desc: null + value: null +include_tokens_per_second: + desc: null + value: false +include_num_input_tokens_seen: + desc: null + value: false +neftune_noise_alpha: + desc: null + value: null +optim_target_modules: + desc: null + value: null diff --git a/wandb/run-20240411_040722-rimyh9a1/files/output.log b/wandb/run-20240411_040722-rimyh9a1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..98d3943f843df0a1561b1ddce759d4ff78d38c7d --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/files/output.log @@ -0,0 +1,6041 @@ +/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:557: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary. + warnings.warn(_create_warning_msg( +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:557: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary. + warnings.warn(_create_warning_msg( +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector. + warnings.warn('Was asked to gather along dimension 0, but all ' +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. + warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.") +/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`. diff --git a/wandb/run-20240411_040722-rimyh9a1/files/requirements.txt b/wandb/run-20240411_040722-rimyh9a1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75d653a4d1ff18898bcd3ec2bda23c15dc7051d9 --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/files/requirements.txt @@ -0,0 +1,864 @@ +Babel==2.14.0 +Boruta==0.3 +Brotli==1.0.9 +CVXcanon==0.1.2 +Cartopy==0.22.0 +Cython==3.0.8 +Deprecated==1.2.14 +Farama-Notifications==0.0.4 +Flask==3.0.2 +Geohash==1.0 +GitPython==3.1.41 +ImageHash==4.3.1 +Janome==0.5.0 +Jinja2==3.1.2 +LunarCalendar==0.0.9 +Mako==1.3.2 +Markdown==3.5.2 +MarkupSafe==2.1.3 +MarkupSafe==2.1.5 +Pillow==9.5.0 +PuLP==2.8.0 +PyArabic==0.6.15 +PyJWT==2.8.0 +PyMeeus==0.5.12 +PySocks==1.7.1 +PyUpSet==0.1.1.post7 +PyWavelets==1.5.0 +PyYAML==6.0.1 +Pygments==2.17.2 +Pympler==1.0.1 +QtPy==2.4.1 +Rtree==1.2.0 +SQLAlchemy==2.0.25 +SecretStorage==3.3.3 +Send2Trash==1.8.2 +Shapely==1.8.5.post1 +Shimmy==1.3.0 +SimpleITK==2.3.1 +TPOT==0.12.1 +Theano-PyMC==1.1.2 +Theano==1.0.5 +Wand==0.6.13 +Werkzeug==3.0.2 +absl-py==1.4.0 +accelerate==0.28.0 +access==1.1.9 +affine==2.4.0 +aiobotocore==2.12.2 +aiofiles==22.1.0 +aiohttp-cors==0.7.0 +aiohttp==3.9.1 +aioitertools==0.11.0 +aiorwlock==1.3.0 +aiosignal==1.3.1 +aiosqlite==0.19.0 +albumentations==1.4.0 +alembic==1.13.1 +altair==5.3.0 +annotated-types==0.6.0 +annoy==1.17.3 +anyio==4.2.0 +apache-beam==2.46.0 +aplus==0.11.0 +appdirs==1.4.4 +archspec==0.2.3 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +array-record==0.5.0 +arrow==1.3.0 +arviz==0.17.1 +astroid==3.1.0 +astropy-iers-data==0.2024.4.1.0.33.14 +astropy==6.0.1 +asttokens==2.4.1 +astunparse==1.6.3 +async-lru==2.0.4 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +autopep8==2.0.4 +backoff==2.2.1 +bayesian-optimization==1.4.3 +beatrix_jupyterlab==2023.128.151533 +beautifulsoup4==4.12.2 +bitsandbytes==0.43.0 +blake3==0.2.1 +bleach==6.1.0 +blessed==1.20.0 +blinker==1.7.0 +blis==0.7.10 +blosc2==2.6.0 +bokeh==3.3.4 +boltons==23.1.1 +boto3==1.26.100 +botocore==1.34.51 +bq_helper==0.4.1 +bqplot==0.12.43 +branca==0.7.1 +brewer2mpl==1.4.1 +brotlipy==0.7.0 +cached-property==1.5.2 +cachetools==4.2.4 +cachetools==5.3.2 +catalogue==2.0.10 +catalyst==22.4 +catboost==1.2.3 +category-encoders==2.6.3 +certifi==2024.2.2 +cesium==0.12.1 +cffi==1.16.0 +charset-normalizer==3.3.2 +chex==0.1.86 +cleverhans==4.0.0 +click-plugins==1.1.1 +click==8.1.7 +cligj==0.7.2 +cloud-tpu-client==0.10 +cloud-tpu-profiler==2.4.0 +cloudpathlib==0.16.0 +cloudpickle==2.2.1 +cloudpickle==3.0.0 +cmdstanpy==1.2.2 +colorama==0.4.6 +colorcet==3.1.0 +colorful==0.5.6 +colorlog==6.8.2 +colorlover==0.3.0 +comm==0.2.1 +conda-libmamba-solver==23.7.0 +conda-package-handling==2.2.0 +conda==23.7.4 +conda_package_streaming==0.9.0 +confection==0.1.4 +contextily==1.6.0 +contourpy==1.2.0 +convertdate==2.4.0 +crcmod==1.7 +cryptography==41.0.7 +cuda-python==12.4.0 +cudf==23.8.0 +cufflinks==0.17.3 +cuml==23.8.0 +cupy==13.0.0 +cycler==0.12.1 +cymem==2.0.8 +cytoolz==0.12.3 +daal4py==2024.2.0 +daal==2024.2.0 +dacite==1.8.1 +dask-cuda==23.8.0 +dask-cudf==23.8.0 +dask-expr==1.0.9 +dask==2024.4.0 +dataclasses-json==0.6.4 +dataproc_jupyter_plugin==0.1.66 +datasets==2.16.0 +datashader==0.16.0 +datatile==1.0.3 +db-dtypes==1.2.0 +deap==1.4.1 +debugpy==1.8.0 +decorator==5.1.1 +deepdiff==6.7.1 +defusedxml==0.7.1 +deprecation==2.1.0 +descartes==1.1.0 +dill==0.3.7 +dipy==1.9.0 +distlib==0.3.8 +distributed==2023.7.1 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docker==7.0.0 +docopt==0.6.2 +docstring-parser==0.15 +docstring-to-markdown==0.15 +docutils==0.20.1 +earthengine-api==0.1.395 +easydict==1.13 +easyocr==1.7.1 +ecos==2.0.13 +eli5==0.13.0 +emoji==2.11.0 +en-core-web-lg==3.7.1 +en-core-web-sm==3.7.1 +entrypoints==0.4 +ephem==4.1.5 +esda==2.5.1 +essentia==2.1b6.dev1110 +et-xmlfile==1.1.0 +etils==1.6.0 +exceptiongroup==1.2.0 +executing==2.0.1 +explainable-ai-sdk==1.3.3 +fastai==2.7.14 +fastapi==0.108.0 +fastavro==1.9.3 +fastcore==1.5.29 +fastdownload==0.0.7 +fasteners==0.19 +fastjsonschema==2.19.1 +fastprogress==1.0.3 +fastrlock==0.8.2 +fasttext==0.9.2 +feather-format==0.4.1 +featuretools==1.30.0 +filelock==3.13.1 +fiona==1.9.6 +fitter==1.7.0 +flake8==7.0.0 +flashtext==2.7 +flatbuffers==23.5.26 +flax==0.8.2 +folium==0.16.0 +fonttools==4.47.0 +fonttools==4.50.0 +fqdn==1.5.1 +frozendict==2.4.1 +frozenlist==1.4.1 +fsspec==2023.10.0 +fsspec==2024.3.1 +funcy==2.0 +fury==0.10.0 +future==1.0.0 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gatspy==0.3 +gcsfs==2024.2.0 +gensim==4.3.2 +geographiclib==2.0 +geojson==3.1.0 +geopandas==0.14.3 +geoplot==0.5.1 +geopy==2.4.1 +geoviews==1.11.1 +ggplot==0.11.5 +giddy==2.3.5 +gitdb==4.0.11 +google-ai-generativelanguage==0.4.0 +google-api-core==2.11.1 +google-api-core==2.18.0 +google-api-python-client==2.125.0 +google-apitools==0.5.31 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-auth==2.26.1 +google-cloud-aiplatform==0.6.0a1 +google-cloud-artifact-registry==1.10.0 +google-cloud-automl==1.0.1 +google-cloud-bigquery==2.34.4 +google-cloud-bigtable==1.7.3 +google-cloud-core==2.4.1 +google-cloud-datastore==2.19.0 +google-cloud-dlp==3.14.0 +google-cloud-jupyter-config==0.0.5 +google-cloud-language==2.13.3 +google-cloud-monitoring==2.18.0 +google-cloud-pubsub==2.19.0 +google-cloud-pubsublite==1.9.0 +google-cloud-recommendations-ai==0.7.1 +google-cloud-resource-manager==1.11.0 +google-cloud-spanner==3.40.1 +google-cloud-storage==1.44.0 +google-cloud-translate==3.12.1 +google-cloud-videointelligence==2.13.3 +google-cloud-vision==2.8.0 +google-crc32c==1.5.0 +google-generativeai==0.4.1 +google-pasta==0.2.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 +gplearn==0.4.2 +gpustat==1.0.0 +gpxpy==1.6.2 +graphviz==0.20.3 +greenlet==3.0.3 +grpc-google-iam-v1==0.12.7 +grpcio-status==1.48.1 +grpcio-status==1.48.2 +grpcio==1.51.1 +grpcio==1.60.0 +gviz-api==1.10.0 +gym-notices==0.0.8 +gym==0.26.2 +gymnasium==0.29.0 +h11==0.14.0 +h2o==3.46.0.1 +h5netcdf==1.3.0 +h5py==3.10.0 +haversine==2.8.1 +hdfs==2.7.3 +hep-ml==0.7.2 +hijri-converter==2.3.1 +hmmlearn==0.3.2 +holidays==0.24 +holoviews==1.18.3 +hpsklearn==0.1.0 +html5lib==1.1 +htmlmin==0.1.12 +httpcore==1.0.5 +httplib2==0.21.0 +httptools==0.6.1 +httpx==0.27.0 +huggingface-hub==0.22.2 +hunspell==0.5.5 +hydra-slayer==0.5.0 +hyperopt==0.2.7 +hypertools==0.8.0 +idna==3.6 +igraph==0.11.4 +imagecodecs==2024.1.1 +imageio==2.33.1 +imbalanced-learn==0.12.2 +imgaug==0.4.0 +importlib-metadata==6.11.0 +importlib-metadata==7.0.1 +importlib-resources==6.1.1 +inequality==1.0.1 +iniconfig==2.0.0 +ipydatawidgets==4.3.5 +ipykernel==6.28.0 +ipyleaflet==0.18.2 +ipympl==0.7.0 +ipython-genutils==0.2.0 +ipython-genutils==0.2.0 +ipython-sql==0.5.0 +ipython==8.20.0 +ipyvolume==0.6.3 +ipyvue==1.10.2 +ipyvuetify==1.9.3 +ipywebrtc==0.6.0 +ipywidgets==7.7.1 +isoduration==20.11.0 +isort==5.13.2 +isoweek==1.3.3 +itsdangerous==2.1.2 +jaraco.classes==3.3.0 +jax-jumpy==1.0.0 +jax==0.4.23 +jaxlib==0.4.23.dev20240116 +jedi==0.19.1 +jeepney==0.8.0 +jieba==0.42.1 +jmespath==1.0.1 +joblib==1.3.2 +json5==0.9.14 +jsonpatch==1.33 +jsonpointer==2.4 +jsonschema-specifications==2023.12.1 +jsonschema==4.20.0 +jupyter-console==6.6.3 +jupyter-events==0.9.0 +jupyter-http-over-ws==0.0.8 +jupyter-lsp==1.5.1 +jupyter-server-mathjax==0.2.6 +jupyter-ydoc==0.2.5 +jupyter_client==7.4.9 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +jupyter_server==2.13.0 +jupyter_server_fileid==0.9.1 +jupyter_server_proxy==4.1.0 +jupyter_server_terminals==0.5.1 +jupyter_server_ydoc==0.8.0 +jupyterlab-lsp==5.1.0 +jupyterlab-widgets==3.0.9 +jupyterlab==4.1.5 +jupyterlab_git==0.44.0 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.25.2 +jupytext==1.16.0 +kaggle-environments==1.14.3 +kaggle==1.6.8 +kagglehub==0.2.2 +keras-cv==0.8.2 +keras-nlp==0.8.2 +keras-tuner==1.4.6 +keras==3.1.1 +kernels-mixer==0.0.7 +keyring==24.3.0 +keyrings.google-artifactregistry-auth==1.1.2 +kfp-pipeline-spec==0.2.2 +kfp-server-api==2.0.5 +kfp==2.5.0 +kiwisolver==1.4.5 +kmapper==2.0.1 +kmodes==0.12.2 +korean-lunar-calendar==0.3.1 +kornia==0.7.2 +kornia_rs==0.1.3 +kt-legacy==1.0.5 +kubernetes==26.1.0 +langcodes==3.3.0 +langid==1.1.6 +lazy_loader==0.3 +learntools==0.3.4 +leven==1.0.4 +libclang==16.0.6 +libmambapy==1.5.0 +libpysal==4.9.2 +librosa==0.10.1 +lightgbm==4.2.0 +lightning-utilities==0.11.2 +lime==0.2.0.1 +line-profiler==4.1.2 +linkify-it-py==2.0.3 +llvmlite==0.41.1 +llvmlite==0.42.0 +lml==0.1.0 +locket==1.0.0 +loguru==0.7.2 +lxml==5.2.1 +lz4==4.3.3 +mamba==1.5.0 +mapclassify==2.6.1 +markdown-it-py==3.0.0 +marshmallow==3.21.1 +matplotlib-inline==0.1.6 +matplotlib-venn==0.11.10 +matplotlib==3.7.5 +matplotlib==3.8.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +memory-profiler==0.61.0 +menuinst==2.0.1 +mercantile==1.2.1 +mgwr==2.2.1 +missingno==0.5.2 +mistune==0.8.4 +mizani==0.11.1 +ml-dtypes==0.2.0 +mlcrate==0.2.0 +mlens==0.2.3 +mlxtend==0.23.1 +mne==1.6.1 +mnist==0.2.2 +momepy==0.7.0 +more-itertools==10.2.0 +mpld3==0.5.10 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multimethod==1.10 +multipledispatch==1.0.0 +multiprocess==0.70.15 +munkres==1.1.4 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +namex==0.0.7 +nb-conda-kernels==2.3.1 +nb_conda==2.2.1 +nbclassic==1.0.0 +nbclient==0.5.13 +nbconvert==6.4.5 +nbdime==3.2.0 +nbformat==5.9.2 +ndindex==1.8 +nest-asyncio==1.5.8 +networkx==3.2.1 +nibabel==5.2.1 +nilearn==0.10.3 +ninja==1.11.1.1 +nltk==3.2.4 +nose==1.3.7 +notebook==6.5.4 +notebook==6.5.6 +notebook_executor==0.2 +notebook_shim==0.2.3 +numba==0.58.1 +numba==0.59.1 +numexpr==2.10.0 +numpy==1.26.4 +nvidia-ml-py==11.495.46 +nvtx==0.2.10 +oauth2client==4.1.3 +oauthlib==3.2.2 +objsize==0.6.1 +odfpy==1.4.1 +olefile==0.47 +onnx==1.16.0 +opencensus-context==0.1.3 +opencensus==0.11.4 +opencv-contrib-python==4.9.0.80 +opencv-python-headless==4.9.0.80 +opencv-python==4.9.0.80 +openpyxl==3.1.2 +openslide-python==1.3.1 +opentelemetry-api==1.22.0 +opentelemetry-exporter-otlp-proto-common==1.22.0 +opentelemetry-exporter-otlp-proto-grpc==1.22.0 +opentelemetry-exporter-otlp-proto-http==1.22.0 +opentelemetry-exporter-otlp==1.22.0 +opentelemetry-proto==1.22.0 +opentelemetry-sdk==1.22.0 +opentelemetry-semantic-conventions==0.43b0 +opt-einsum==3.3.0 +optax==0.2.2 +optree==0.11.0 +optuna==3.6.1 +orbax-checkpoint==0.5.7 +ordered-set==4.1.0 +orjson==3.9.10 +ortools==9.4.1874 +osmnx==1.9.2 +overrides==7.4.0 +packaging==21.3 +pandas-datareader==0.10.0 +pandas-profiling==3.6.6 +pandas-summary==0.2.0 +pandas==2.1.4 +pandas==2.2.1 +pandasql==0.7.3 +pandocfilters==1.5.0 +panel==1.3.8 +papermill==2.5.0 +param==2.1.0 +parso==0.8.3 +partd==1.4.1 +path.py==12.5.0 +path==16.10.0 +pathos==0.3.2 +pathy==0.10.3 +patsy==0.5.6 +pdf2image==1.17.0 +peft==0.10.0 +pettingzoo==1.24.0 +pexpect==4.8.0 +pexpect==4.9.0 +phik==0.12.4 +pickleshare==0.7.5 +pillow==10.3.0 +pip==23.3.2 +pkgutil_resolve_name==1.3.10 +platformdirs==4.2.0 +plotly-express==0.4.1 +plotly==5.18.0 +plotnine==0.13.4 +pluggy==1.4.0 +pointpats==2.4.0 +polars==0.20.18 +polyglot==16.7.4 +pooch==1.8.1 +pox==0.3.4 +ppca==0.0.4 +ppft==1.7.6.8 +preprocessing==0.1.13 +preshed==3.0.9 +prettytable==3.9.0 +progressbar2==4.4.2 +prometheus-client==0.19.0 +promise==2.3 +prompt-toolkit==3.0.42 +prompt-toolkit==3.0.43 +prophet==1.1.1 +proto-plus==1.23.0 +protobuf==3.20.3 +protobuf==4.21.12 +psutil==5.9.3 +psutil==5.9.7 +ptyprocess==0.7.0 +pudb==2024.1 +pure-eval==0.2.2 +py-cpuinfo==9.0.0 +py-spy==0.3.14 +py4j==0.10.9.7 +pyLDAvis==3.4.1 +pyOpenSSL==23.3.0 +pyaml==23.12.0 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11==2.12.0 +pyclipper==1.3.0.post5 +pycodestyle==2.11.1 +pycosat==0.6.6 +pycparser==2.21 +pycryptodome==3.20.0 +pyct==0.5.0 +pycuda==2024.1 +pydantic==2.5.3 +pydantic==2.6.4 +pydantic_core==2.14.6 +pydantic_core==2.16.3 +pydegensac==0.1.2 +pydicom==2.4.4 +pydocstyle==6.3.0 +pydot==1.4.2 +pydub==0.25.1 +pyemd==1.0.0 +pyerfa==2.0.1.1 +pyexcel-io==0.6.6 +pyexcel-ods==0.6.0 +pyflakes==3.2.0 +pygltflib==1.16.2 +pykalman==0.9.7 +pylibraft==23.8.0 +pylint==3.1.0 +pymc3==3.11.4 +pymongo==3.13.0 +pynndescent==0.5.12 +pynvml==11.4.1 +pynvrtc==9.2 +pyparsing==3.1.1 +pyparsing==3.1.2 +pypdf==4.1.0 +pyproj==3.6.1 +pysal==24.1 +pyshp==2.3.1 +pytesseract==0.3.10 +pytest==8.1.1 +python-bidi==0.4.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.0 +python-json-logger==2.0.7 +python-louvain==0.16 +python-lsp-jsonrpc==1.1.2 +python-lsp-server==1.11.0 +python-slugify==8.0.4 +python-utils==3.8.2 +pythreejs==2.4.2 +pytoolconfig==1.3.1 +pytools==2024.1.1 +pytorch-ignite==0.5.0.post2 +pytorch-lightning==2.2.1 +pytz==2023.3.post1 +pytz==2024.1 +pyu2f==0.1.5 +pyviz_comms==3.0.2 +pyzmq==24.0.1 +pyzmq==25.1.2 +qgrid==1.3.1 +qtconsole==5.5.1 +quantecon==0.7.2 +qudida==0.0.4 +raft-dask==23.8.0 +rasterio==1.3.9 +rasterstats==0.19.0 +ray-cpp==2.9.0 +ray==2.9.0 +referencing==0.32.1 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==0.10.1 +requests==2.31.0 +retrying==1.3.3 +retrying==1.3.4 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rgf-python==3.12.0 +rich-click==1.7.4 +rich==13.7.0 +rich==13.7.1 +rmm==23.8.0 +rope==1.13.0 +rpds-py==0.16.2 +rsa==4.9 +ruamel-yaml-conda==0.15.100 +ruamel.yaml.clib==0.2.7 +ruamel.yaml==0.17.40 +s2sphere==0.2.5 +s3fs==2024.2.0 +s3transfer==0.6.2 +safetensors==0.4.2 +scattertext==0.1.19 +scikit-image==0.22.0 +scikit-learn-intelex==2024.2.0 +scikit-learn==1.2.2 +scikit-multilearn==0.2.0 +scikit-optimize==0.10.1 +scikit-plot==0.3.7 +scikit-surprise==1.1.3 +scipy==1.11.4 +scipy==1.12.0 +seaborn==0.12.2 +segment_anything==1.0 +segregation==2.5 +semver==3.0.2 +sentencepiece==0.2.0 +sentry-sdk==1.44.1 +setproctitle==1.3.3 +setuptools-git==1.2 +setuptools-scm==8.0.4 +setuptools==69.0.3 +shap==0.44.1 +shapely==2.0.3 +shellingham==1.5.4 +shtab==1.7.1 +simpervisor==1.0.0 +simplejson==3.19.2 +six==1.16.0 +sklearn-pandas==2.2.0 +slicer==0.0.7 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +spaghetti==1.7.5.post1 +spectral==0.23.1 +spglm==1.1.0 +sphinx-rtd-theme==0.2.4 +spint==1.0.7 +splot==1.1.5.post1 +spopt==0.6.0 +spreg==1.4.2 +spvcm==0.3.0 +sqlparse==0.4.4 +squarify==0.4.3 +srsly==2.4.8 +stable-baselines3==2.1.0 +stack-data==0.6.2 +stack-data==0.6.3 +stanio==0.5.0 +starlette==0.32.0.post1 +statsmodels==0.14.1 +stemming==1.0.1 +stop-words==2018.7.23 +stopit==1.1.2 +stumpy==1.12.0 +sympy==1.12 +tables==3.9.2 +tabulate==0.9.0 +tangled-up-in-unicode==0.2.0 +tbb==2021.12.0 +tblib==3.0.0 +tenacity==8.2.3 +tensorboard-data-server==0.7.2 +tensorboard-plugin-profile==2.15.0 +tensorboard==2.15.1 +tensorboardX==2.6.2.2 +tensorflow-cloud==0.1.16 +tensorflow-datasets==4.9.4 +tensorflow-decision-forests==1.8.1 +tensorflow-estimator==2.15.0 +tensorflow-hub==0.16.1 +tensorflow-io-gcs-filesystem==0.35.0 +tensorflow-io==0.35.0 +tensorflow-metadata==0.14.0 +tensorflow-probability==0.23.0 +tensorflow-serving-api==2.14.1 +tensorflow-text==2.15.0 +tensorflow-transform==0.14.0 +tensorflow==2.15.0 +tensorstore==0.1.56 +termcolor==2.4.0 +terminado==0.18.0 +testpath==0.6.0 +text-unidecode==1.3 +textblob==0.18.0.post0 +texttable==1.7.0 +tf_keras==2.15.1 +tfp-nightly==0.24.0.dev0 +thinc==8.2.2 +threadpoolctl==3.2.0 +tifffile==2023.12.9 +timm==0.9.16 +tinycss2==1.2.1 +tobler==0.11.2 +tokenizers==0.15.2 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.12.4 +toolz==0.12.1 +torch==2.1.2 +torchaudio==2.1.2 +torchdata==0.7.1 +torchinfo==1.8.0 +torchmetrics==1.3.2 +torchtext==0.16.2 +torchvision==0.16.2 +tornado==6.3.3 +tqdm==4.66.1 +traceml==1.0.8 +traitlets==5.9.0 +traittypes==0.2.1 +transformers==4.39.3 +treelite-runtime==3.2.0 +treelite==3.2.0 +trl==0.8.1 +truststore==0.8.0 +trx-python==0.2.9 +tsfresh==0.20.2 +typeguard==4.1.5 +typer==0.9.0 +typer==0.9.4 +types-python-dateutil==2.8.19.20240106 +typing-inspect==0.9.0 +typing-utils==0.1.0 +typing_extensions==4.9.0 +tyro==0.8.3 +tzdata==2023.4 +uc-micro-py==1.0.3 +ucx-py==0.33.0 +ujson==5.9.0 +umap-learn==0.5.5 +unicodedata2==15.1.0 +update-checker==0.18.0 +uri-template==1.3.0 +uritemplate==3.0.1 +urllib3==1.26.18 +urllib3==2.1.0 +urwid==2.6.10 +urwid_readline==0.14 +uvicorn==0.25.0 +uvloop==0.19.0 +vaex-astro==0.9.3 +vaex-core==4.17.1 +vaex-hdf5==0.14.1 +vaex-jupyter==0.8.2 +vaex-ml==0.18.3 +vaex-server==0.9.0 +vaex-viz==0.5.4 +vaex==4.17.0 +vec_noise==1.1.4 +vecstack==0.4.0 +virtualenv==20.21.0 +visions==0.7.5 +vowpalwabbit==9.9.0 +vtk==9.3.0 +wandb==0.16.5 +wasabi==1.1.2 +watchfiles==0.21.0 +wavio==0.0.8 +wcwidth==0.2.13 +weasel==0.3.4 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.7.0 +websockets==12.0 +wfdb==4.1.2 +whatthepatch==1.0.5 +wheel==0.42.0 +widgetsnbextension==3.6.6 +witwidget==1.8.1 +woodwork==0.29.0 +wordcloud==1.9.3 +wordsegment==1.3.1 +wrapt==1.14.1 +xarray-einstats==0.7.0 +xarray==2024.3.0 +xgboost==2.0.3 +xvfbwrapper==0.2.9 +xxhash==3.4.1 +xyzservices==2023.10.1 +y-py==0.6.2 +yapf==0.40.2 +yarl==1.9.3 +yarl==1.9.4 +ydata-profiling==4.6.4 +yellowbrick==1.5 +ypy-websocket==0.8.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.22.0 \ No newline at end of file diff --git a/wandb/run-20240411_040722-rimyh9a1/files/wandb-metadata.json b/wandb/run-20240411_040722-rimyh9a1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5984be6d5e330df393ff78a40e21a1a6ec994625 --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/files/wandb-metadata.json @@ -0,0 +1,66 @@ +{ + "os": "Linux-5.15.133+-x86_64-with-glibc2.31", + "python": "3.10.13", + "heartbeatAt": "2024-04-11T04:07:23.361534", + "startedAt": "2024-04-11T04:07:22.840819", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "kaggle.ipynb", + "codePathLocal": null, + "root": "/kaggle/working", + "host": "321ddc84c049", + "username": "root", + "executable": "/opt/conda/bin/python3.10", + "cpu_count": 2, + "cpu_count_logical": 4, + "cpu_freq": { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2000.19, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 8062.387607574463, + "used": 5566.962131500244 + } + }, + "gpu": "Tesla T4", + "gpu_count": 2, + "gpu_devices": [ + { + "name": "Tesla T4", + "memory_total": 16106127360 + }, + { + "name": "Tesla T4", + "memory_total": 16106127360 + } + ], + "memory": { + "total": 31.357559204101562 + } +} diff --git a/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json b/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7b169b318962594d0d41fd667f8df6a00ad8023f --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss": 3.4153, "train/grad_norm": 0.0, "train/learning_rate": 0.0, "train/epoch": 0.09, "train/global_step": 20000, "_timestamp": 1712827654.7326186, "_runtime": 19211.883972644806, "_step": 1006, "train_runtime": 18957.1753, "train_samples_per_second": 2.11, "train_steps_per_second": 1.055, "total_flos": 1.0807103426248704e+16, "train_loss": 3.9423419631004335} \ No newline at end of file diff --git a/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log b/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a2d91a1066637700385639e51652426782b2958a --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log @@ -0,0 +1,15349 @@ +2024-04-11 04:07:22,850 INFO StreamThr :1091 [internal.py:wandb_internal():86] W&B internal server running at pid: 1091, started at: 2024-04-11 04:07:22.849871 +2024-04-11 04:07:22,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status +2024-04-11 04:07:23,106 INFO WriterThread:1091 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/run-rimyh9a1.wandb +2024-04-11 04:07:23,107 DEBUG SenderThread:1091 [sender.py:send():379] send: header +2024-04-11 04:07:23,109 DEBUG SenderThread:1091 [sender.py:send():379] send: run +2024-04-11 04:07:23,229 INFO SenderThread:1091 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files +2024-04-11 04:07:23,229 INFO SenderThread:1091 [sender.py:_start_run_threads():1124] run started: rimyh9a1 with start time 1712808442.848646 +2024-04-11 04:07:23,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: check_version +2024-04-11 04:07:23,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: check_version +2024-04-11 04:07:23,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: run_start +2024-04-11 04:07:23,341 DEBUG HandlerThread:1091 [system_info.py:__init__():26] System info init +2024-04-11 04:07:23,341 DEBUG HandlerThread:1091 [system_info.py:__init__():41] System info init done +2024-04-11 04:07:23,341 INFO HandlerThread:1091 [system_monitor.py:start():194] Starting system monitor +2024-04-11 04:07:23,341 INFO SystemMonitor:1091 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-11 04:07:23,342 INFO HandlerThread:1091 [system_monitor.py:probe():214] Collecting system info +2024-04-11 04:07:23,342 INFO SystemMonitor:1091 [interfaces.py:start():190] Started cpu monitoring +2024-04-11 04:07:23,343 INFO SystemMonitor:1091 [interfaces.py:start():190] Started disk monitoring +2024-04-11 04:07:23,344 INFO SystemMonitor:1091 [interfaces.py:start():190] Started gpu monitoring +2024-04-11 04:07:23,345 INFO SystemMonitor:1091 [interfaces.py:start():190] Started memory monitoring +2024-04-11 04:07:23,346 INFO SystemMonitor:1091 [interfaces.py:start():190] Started network monitoring +2024-04-11 04:07:23,361 DEBUG HandlerThread:1091 [system_info.py:probe():150] Probing system +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_info.py:probe():198] Probing system done +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-11T04:07:23.361534', 'startedAt': '2024-04-11T04:07:22.840819', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '321ddc84c049', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.19, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5566.962131500244}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-04-11 04:07:23,364 INFO HandlerThread:1091 [system_monitor.py:probe():224] Finished collecting system info +2024-04-11 04:07:23,364 INFO HandlerThread:1091 [system_monitor.py:probe():227] Publishing system info +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-11 04:07:24,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/conda-environment.yaml +2024-04-11 04:07:38,380 ERROR HandlerThread:1091 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +Traceback (most recent call last): + File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda + subprocess.call( + File "/opt/conda/lib/python3.10/subprocess.py", line 347, in call + return p.wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1209, in wait + return self._wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait + raise TimeoutExpired(self.args, timeout) +subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-04-11 04:07:38,381 DEBUG HandlerThread:1091 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-11 04:07:38,381 INFO HandlerThread:1091 [system_monitor.py:probe():229] Finished publishing system info +2024-04-11 04:07:38,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,388 DEBUG SenderThread:1091 [sender.py:send():379] send: files +2024-04-11 04:07:38,388 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-11 04:07:38,590 INFO wandb-upload_0:1091 [upload_job.py:push():131] Uploaded file /tmp/tmppwkfxjn1wandb/7mpz5val-wandb-metadata.json +2024-04-11 04:07:39,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-metadata.json +2024-04-11 04:07:39,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: python_packages +2024-04-11 04:07:39,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: python_packages +2024-04-11 04:07:39,319 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,329 DEBUG SenderThread:1091 [sender.py:send():379] send: config +2024-04-11 04:07:39,330 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:07:39,331 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,332 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:07:39,332 WARNING SenderThread:1091 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-11 04:07:39,332 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:07:39,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:07:39,335 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:07:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/requirements.txt +2024-04-11 04:07:42,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:43,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:44,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:46,236 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:48,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:53,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:54,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 04:07:55,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:07:55,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:07:55,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:07:59,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:04,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:09,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:09,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:09,452 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:09,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:15,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:20,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:23,346 DEBUG SystemMonitor:1091 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-11 04:08:23,348 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:08:24,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:24,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:24,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:26,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:28,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:08:28,054 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:08:28,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:08:30,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:08:31,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:34,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:08:36,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:39,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:39,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:39,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:42,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:47,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:52,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:53,349 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:08:54,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:54,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:54,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:58,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:59,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 04:09:03,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:08,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:09,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:09,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:09,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:12,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:09:12,735 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:09:12,735 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:09:12,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:09:13,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:09:13,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:14,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:18,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:19,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:23,350 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:09:24,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:24,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:24,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:24,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:30,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:35,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:39,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:39,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:39,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:41,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:09:41,116 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:09:41,116 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:09:41,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:41,119 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:09:41,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:09:44,285 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:46,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:46,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:51,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:53,351 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:09:54,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:54,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:54,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:57,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:02,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:07,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:09,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:09,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:09,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:10,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:10:10,747 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:10:10,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:10:10,750 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:10:11,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:10:12,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:12,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:16,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:17,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:22,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:23,352 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:10:24,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:24,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:24,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:28,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:33,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:38,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:39,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:39,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:39,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:40,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:10:40,953 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:10:40,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:10:40,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:10:41,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:10:42,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:43,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:46,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:49,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:53,353 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:10:54,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:54,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:54,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:55,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:00,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:05,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:09,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:09,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:09,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:11,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:15,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:11:15,551 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:11:15,551 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:11:15,553 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:11:16,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:11:16,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:18,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:20,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:22,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:22,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:23,354 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:11:24,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:24,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:24,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:28,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:33,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:38,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:39,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:39,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:39,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:40,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:44,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:51,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:11:51,548 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:11:51,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:11:51,549 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:11:52,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:11:53,354 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:11:54,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:54,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:54,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:54,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:54,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:56,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:00,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:05,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:05,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:05,936 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:05,937 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:05,937 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:06,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:08,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:09,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:09,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:09,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:10,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:11,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:16,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:21,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:23,355 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:12:24,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:24,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:24,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:27,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:31,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:31,049 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:31,049 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:31,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:31,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:32,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:33,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:36,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:38,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:39,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:39,486 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:39,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:44,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:47,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:47,842 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:47,842 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:47,842 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:48,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:50,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:52,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:53,356 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:12:54,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:54,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:54,491 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:55,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:00,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:05,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:07,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:07,619 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:07,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:07,619 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:08,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:10,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:10,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:10,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:10,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:10,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:12,368 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:15,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:20,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:21,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:21,256 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:21,256 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:21,258 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:21,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:22,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:23,358 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:13:25,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:25,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:25,210 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:26,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:26,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:31,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:36,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:39,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:39,561 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:39,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:39,562 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:40,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:40,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:40,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:40,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:42,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:42,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:44,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:47,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:52,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:53,358 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:13:55,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:55,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:55,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:55,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:55,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:55,394 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:55,395 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:56,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:58,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:58,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:00,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:03,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:08,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:10,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:10,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:14,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:15,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:15,976 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:15,977 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:15,977 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:16,399 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:18,400 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:20,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:20,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:23,360 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:14:25,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:25,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:25,213 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:25,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:29,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:29,755 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:29,756 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:29,756 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:30,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:30,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:32,406 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:34,407 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:36,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:40,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:41,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:46,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:49,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:49,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:49,456 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:49,456 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:50,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:51,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:52,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:53,360 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:14:54,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:55,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:55,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:55,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:57,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:02,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:05,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:05,935 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:05,936 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:05,936 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:06,418 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:07,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:08,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:10,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:10,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:13,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:18,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:22,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:22,440 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:22,441 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:22,441 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:23,361 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:15:23,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:23,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:24,425 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:25,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:25,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:25,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:26,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:28,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:28,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:33,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:38,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:40,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:40,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:40,339 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:40,339 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:40,340 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:40,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:42,432 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:44,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:44,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:49,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:53,362 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:15:55,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:55,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:55,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:55,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:57,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:57,763 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:57,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:57,764 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:58,438 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:00,439 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:00,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:02,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:06,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:10,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:10,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:11,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:16,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:16,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:16,568 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:16,568 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:16,568 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:17,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:18,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:21,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:22,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:23,363 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:16:25,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:25,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:25,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:27,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:32,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:32,171 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:32,171 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:32,172 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:32,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:33,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:34,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:36,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:38,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:40,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:40,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:40,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:44,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:49,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:53,364 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:16:54,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:54,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:54,714 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:54,714 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:54,715 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:55,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:55,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:55,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:55,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:56,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:59,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:00,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:04,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:09,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:10,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:10,219 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:10,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:11,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:11,031 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:11,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:11,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:11,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:12,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:15,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:16,468 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:20,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:23,365 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:17:25,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:25,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:25,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:25,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:28,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:28,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:28,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:28,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:28,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:30,473 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:31,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:34,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:37,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:40,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:40,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:42,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:46,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:46,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:46,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:46,044 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:46,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:48,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:48,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:50,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:53,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:53,367 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:17:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:55,219 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:58,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:03,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:05,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:05,870 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:05,870 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:05,871 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:06,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:08,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:08,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:10,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:10,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:10,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:10,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:14,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:19,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:23,367 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:18:24,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:24,895 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:24,896 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:24,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:24,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:25,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:25,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:25,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:25,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:26,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:30,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:30,496 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:35,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:40,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:40,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:40,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:45,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:46,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:46,066 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:46,066 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:46,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:46,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:48,503 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:50,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:50,504 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:53,368 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:18:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:55,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:55,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:56,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:01,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:04,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:04,875 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:04,875 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:04,876 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:05,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:06,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:06,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:10,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:10,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:10,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:10,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:12,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:17,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:22,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:22,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:22,759 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:22,759 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:22,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:23,369 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:19:23,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:24,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:25,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:25,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:25,352 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:26,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:27,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:28,518 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:32,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:37,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:39,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:39,841 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:39,841 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:39,842 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:40,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:40,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:40,350 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:40,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:40,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:42,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:44,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:47,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:52,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:53,370 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:19:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:55,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:58,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:00,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:00,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:00,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:00,025 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:00,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:02,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:04,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:04,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:09,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:10,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:10,350 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:10,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:14,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:19,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:19,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:19,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:19,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:19,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:20,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:20,537 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:23,371 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:20:24,538 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:25,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:25,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:25,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:25,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:30,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:35,422 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:36,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:36,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:36,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:36,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:36,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:38,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:40,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:40,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:40,480 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:40,544 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:45,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:50,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:52,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:52,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:52,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:52,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:52,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:53,372 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:20:54,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:55,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:55,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:55,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:55,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:56,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:00,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:05,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:10,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:10,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:10,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:10,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:11,000 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:11,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:11,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:11,001 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:11,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:12,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:14,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:16,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:16,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:21,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:23,373 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:21:25,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:25,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:25,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:26,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:31,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:32,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:32,406 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:32,406 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:32,406 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:32,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:34,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:36,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:36,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:40,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:40,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:40,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:42,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:47,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:52,419 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:53,374 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:21:55,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:55,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:55,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:57,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:57,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:57,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:57,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:57,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:58,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:58,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:02,580 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:03,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:08,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:10,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:10,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:10,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:13,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:18,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:21,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:21,794 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:21,794 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:21,795 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:22,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:22,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:23,375 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:22:24,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:25,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:25,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:25,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:26,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:29,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:34,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:36,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:36,442 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:36,442 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:36,443 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:36,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:38,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:39,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:40,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:40,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:40,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:40,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:45,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:50,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:53,376 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:22:55,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:55,088 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:55,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:55,088 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:55,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:55,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:55,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:55,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:55,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:56,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:58,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:00,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:01,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:06,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:10,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:10,357 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:10,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:11,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:16,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:18,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:18,859 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:18,859 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:18,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:19,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:20,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:21,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:22,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:23,377 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:23:25,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:25,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:25,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:27,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:32,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:35,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:35,184 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:35,185 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:35,185 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:35,614 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:36,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:38,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:38,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:40,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:40,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:40,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:40,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:43,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:48,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:52,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:52,722 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:52,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:52,723 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:53,378 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:23:53,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:54,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:54,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:55,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:55,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:56,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:59,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:04,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:09,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:10,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:10,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:13,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:13,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:13,435 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:13,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:13,642 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:15,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:19,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:20,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:23,380 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:24:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:25,358 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:25,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:26,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:31,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:31,269 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:31,269 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:31,269 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:31,648 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:32,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:33,649 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:35,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:37,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:37,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:40,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:40,358 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:40,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:42,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:47,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:48,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:48,190 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:48,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:48,190 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:48,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:49,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:53,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:53,380 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:24:53,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:55,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:55,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:58,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:03,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:04,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:04,099 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:04,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:04,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:04,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:05,661 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:09,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:09,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:10,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:10,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:10,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:14,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:19,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:19,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:19,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:19,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:19,822 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:20,666 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:21,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:23,381 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:25:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:25,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:25,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:25,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:25,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:30,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:35,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:40,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:40,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:40,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:40,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:40,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:40,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:40,669 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:40,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:43,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:45,676 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:45,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:50,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:53,382 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:25:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:55,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:55,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:56,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:56,384 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:56,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:56,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:56,386 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:56,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:57,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:01,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:01,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:06,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:10,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:10,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:11,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:11,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:11,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:11,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:11,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:12,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:13,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:16,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:17,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:21,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:23,383 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:26:25,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:25,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:25,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:26,636 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:26,637 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:26,638 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:26,638 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:26,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:27,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:29,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:31,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:32,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:37,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:40,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:40,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:43,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:48,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:49,130 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:49,132 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:49,132 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:49,132 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:49,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:51,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:53,384 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:26:53,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:54,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:55,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:55,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:55,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:59,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:04,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:05,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:05,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:05,055 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:05,055 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:05,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:07,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:09,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:10,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:10,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:10,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:15,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:20,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:20,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:20,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:20,616 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:20,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:20,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:23,386 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:27:23,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:25,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:25,363 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:25,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:25,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:26,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:31,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:36,416 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:39,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:39,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:39,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:39,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:39,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:40,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:40,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:40,949 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:41,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:42,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:43,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:47,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:52,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:54,276 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:27:55,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:55,948 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:55,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:59,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:59,294 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:59,294 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:59,294 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:59,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:01,726 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:03,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:03,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:08,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:11,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:11,126 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:11,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:14,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:16,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:16,788 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:16,788 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:16,789 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:17,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:19,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:19,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:21,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:23,388 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:28:25,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:26,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:26,078 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:26,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:31,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:33,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:33,817 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:33,818 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:33,819 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:34,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:35,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:36,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:39,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:41,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:41,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:41,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:42,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:47,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:52,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:53,388 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:28:54,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:54,538 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:54,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:54,539 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:54,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:55,747 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:56,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:56,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:56,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:57,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:59,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:02,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:07,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:11,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:11,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:11,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:13,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:13,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:13,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:13,455 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:13,457 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:13,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:15,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:17,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:18,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:23,390 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:29:24,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:26,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:29,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:31,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:31,362 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:31,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:31,362 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:31,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:33,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:34,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:35,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:37,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:39,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:41,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:41,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:41,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:45,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:50,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:53,391 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:29:55,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:56,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:56,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:56,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:56,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:56,404 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:56,405 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:56,405 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:56,773 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:57,773 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:00,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:01,775 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:05,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:10,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:11,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:11,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:11,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:16,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:18,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:18,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:18,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:18,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:18,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:19,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:21,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:23,392 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:30:23,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:26,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:26,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:27,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:32,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:37,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:37,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:37,742 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:37,742 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:37,742 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:37,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:39,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:41,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:41,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:41,280 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:41,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:42,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:47,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:52,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:53,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:53,327 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:53,327 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:53,328 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:53,393 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:30:53,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:55,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:56,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:56,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:56,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:57,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:57,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:02,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:07,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:09,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:09,124 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:09,124 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:09,125 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:09,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:11,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:11,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:11,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:11,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:13,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:13,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:18,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:23,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:23,394 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:31:26,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:26,747 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:26,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:28,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:30,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:30,670 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:30,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:30,671 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:30,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:31,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:34,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:35,811 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:39,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:41,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:41,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:41,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:45,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:48,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:48,681 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:48,682 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:48,682 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:48,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:49,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:51,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:53,395 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:31:53,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:56,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:56,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:56,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:56,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:01,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:03,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:03,003 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:03,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:03,004 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:03,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:05,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:07,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:07,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:11,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:11,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:11,750 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:12,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:17,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:20,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:20,690 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:20,690 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:20,691 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:20,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:21,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:23,396 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:32:23,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:25,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:26,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:26,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:26,747 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:28,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:33,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:38,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:41,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:41,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:41,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:42,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:42,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:42,485 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:42,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:42,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:43,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:44,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:47,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:49,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:53,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:32:55,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:56,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:56,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:00,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:01,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:01,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:01,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:01,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:01,845 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:03,846 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:05,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:06,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:11,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:11,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:16,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:19,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:19,151 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:19,152 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:19,152 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:19,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:21,853 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:22,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:23,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:33:23,854 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:26,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:26,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:26,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:27,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:32,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:36,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:36,029 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:36,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:36,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:36,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:37,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:38,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:39,861 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:41,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:43,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:48,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:53,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:33:54,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:56,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:56,467 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:56,467 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:56,467 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:56,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:56,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:56,867 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:57,868 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:59,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:01,869 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:04,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:09,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:11,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:11,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:14,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:16,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:16,244 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:16,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:16,246 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:16,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:17,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:20,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:21,876 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:23,403 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:34:25,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:26,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:26,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:26,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:30,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:32,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:32,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:32,979 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:32,979 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:33,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:33,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:35,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:37,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:41,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:41,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:46,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:51,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:53,405 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:34:55,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:55,216 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:55,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:55,217 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:55,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:55,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:56,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:56,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:56,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:56,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:59,891 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:02,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:07,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:11,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:11,936 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:11,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:13,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:13,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:13,231 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:13,232 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:13,232 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:13,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:13,897 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:17,898 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:18,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:23,408 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:35:24,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:26,962 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:29,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:34,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:37,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:37,361 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:37,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:37,362 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:37,905 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:39,906 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:40,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:41,906 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:41,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:41,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:41,941 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:46,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:51,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:53,409 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:35:56,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:56,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:56,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:56,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:58,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:58,531 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:58,531 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:58,532 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:58,913 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:59,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:01,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:03,916 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:06,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:11,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:11,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:11,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:11,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:15,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:15,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:15,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:15,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:15,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:15,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:17,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:19,922 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:22,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:23,410 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:36:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:26,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:26,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:28,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:33,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:35,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:35,940 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:35,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:35,941 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:36,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:37,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:38,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:39,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:41,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:41,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:44,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:49,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:53,411 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:36:54,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:54,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:54,674 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:54,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:54,675 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:54,935 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:55,936 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:57,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:57,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:57,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:59,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:59,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:04,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:09,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:12,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:12,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:15,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:17,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:17,851 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:17,851 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:17,852 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:17,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:19,947 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:20,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:21,948 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:23,411 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:37:26,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:27,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:27,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:32,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:32,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:32,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:32,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:32,219 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:32,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:33,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:35,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:37,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:42,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:42,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:42,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:47,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:49,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:49,329 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:49,329 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:49,330 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:49,959 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:49,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:52,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:53,417 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:37:53,961 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:57,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:58,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:03,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:08,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:08,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:08,410 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:08,410 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:08,410 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:08,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:09,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:11,968 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:12,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:12,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:12,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:13,969 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:14,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:19,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:22,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:22,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:22,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:22,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:23,413 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:38:23,973 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:24,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:25,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:27,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:27,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:27,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:30,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:35,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:38,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:38,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:38,046 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:38,047 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:38,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:39,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:41,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:41,980 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:42,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:42,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:46,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:51,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:53,414 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:38:56,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:57,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:57,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:59,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:59,577 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:59,578 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:59,580 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:59,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:59,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:01,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:03,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:06,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:11,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:12,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:12,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:14,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:14,583 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:14,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:14,583 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:14,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:15,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:17,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:19,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:22,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:23,415 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:39:27,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:27,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:28,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:33,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:37,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:37,112 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:37,112 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:37,115 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:38,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:38,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:39,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:42,002 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:42,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:42,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:42,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:44,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:49,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:53,416 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:39:53,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:53,764 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:53,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:53,765 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:54,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:54,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:56,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:57,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:57,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:57,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:58,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:00,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:05,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:10,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:11,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:11,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:11,777 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:11,779 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:12,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:12,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:12,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:12,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:14,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:15,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:16,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:20,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:23,417 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:40:26,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:27,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:27,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:31,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:33,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:33,480 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:33,480 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:33,481 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:34,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:34,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:36,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:38,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:41,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:42,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:42,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:47,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:50,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:50,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:50,443 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:50,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:51,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:52,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:52,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:53,418 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:40:54,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:57,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:57,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:57,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:58,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:03,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:07,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:07,917 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:07,917 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:07,917 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:08,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:08,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:10,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:12,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:12,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:12,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:14,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:19,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:23,419 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:41:24,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:27,280 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:27,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:27,791 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:27,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:27,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:28,043 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:29,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:33,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:34,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:39,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:42,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:42,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:42,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:45,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:47,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:47,093 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:47,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:47,094 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:48,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:49,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:51,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:53,054 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:53,419 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:41:56,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:57,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:57,279 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:02,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:02,014 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:02,014 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:02,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:02,015 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:02,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:05,058 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:07,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:07,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:12,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:12,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:12,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:17,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:17,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:17,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:17,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:17,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:18,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:19,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:22,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:23,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:23,420 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:42:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:27,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:27,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:28,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:33,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:36,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:36,110 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:36,110 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:36,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:37,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:39,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:39,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:41,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:42,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:42,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:42,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:44,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:49,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:52,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:54,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:55,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:55,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:55,510 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:55,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:42:56,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:57,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:57,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:59,079 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:59,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:04,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:09,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:12,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:12,283 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:12,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:12,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:12,825 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:12,826 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:12,826 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:13,085 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:14,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:15,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:17,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:19,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:20,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:23,422 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:43:25,423 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:27,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:30,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:30,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:30,047 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:30,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:30,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:31,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:33,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:35,094 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:36,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:41,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:42,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:42,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:42,285 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:46,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:51,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:52,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:52,276 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:52,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:52,277 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:53,100 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:53,422 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:43:55,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:56,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:57,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:57,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:57,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:57,287 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:02,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:07,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:08,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:08,707 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:08,707 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:08,710 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:09,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:11,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:12,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:12,287 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:12,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:12,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:13,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:17,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:22,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:23,423 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:44:25,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:25,204 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:25,204 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:25,204 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:26,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:27,113 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:27,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:27,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:27,807 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:27,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:31,114 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:33,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:38,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:42,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:42,671 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:42,671 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:42,673 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:42,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:42,805 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:42,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:43,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:43,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:45,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:47,120 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:49,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:53,424 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:44:55,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:57,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:57,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:57,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:00,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:02,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:02,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:02,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:02,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:03,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:06,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:07,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:11,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:12,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:12,807 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:12,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:16,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:21,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:23,425 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:45:25,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:25,457 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:25,457 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:25,458 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:26,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:27,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:27,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:28,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:28,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:28,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:31,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:32,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:37,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:42,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:43,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:43,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:43,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:46,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:46,088 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:46,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:46,089 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:46,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:47,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:48,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:51,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:53,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:53,426 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:45:58,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:58,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:58,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:59,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:04,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:09,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:10,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:10,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:10,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:10,992 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:11,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:13,152 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:13,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:15,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:15,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:20,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:23,427 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:46:25,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:25,188 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:25,188 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:25,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:25,189 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:26,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:27,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:28,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:28,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:28,592 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:29,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:30,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:31,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:35,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:40,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:43,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:43,585 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:44,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:44,209 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:44,209 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:44,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:45,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:45,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:46,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:49,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:51,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:53,428 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:46:56,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:58,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:58,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:58,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:00,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:00,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:00,428 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:00,428 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:01,170 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:01,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:03,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:05,172 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:06,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:11,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:13,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:13,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:13,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:16,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:17,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:17,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:17,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:17,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:18,176 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:19,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:21,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:23,178 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:23,429 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:47:27,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:28,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:28,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:32,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:35,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:35,720 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:35,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:35,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:36,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:37,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:37,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:41,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:43,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:43,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:43,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:48,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:53,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:55,387 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:47:58,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:58,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:58,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:59,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:00,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:00,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:00,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:00,242 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:01,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:01,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:05,194 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:05,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:10,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:13,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:13,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:13,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:15,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:17,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:17,898 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:17,899 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:17,899 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:18,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:19,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:20,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:23,201 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:23,430 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:48:26,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:28,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:28,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:31,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:31,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:31,943 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:31,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:31,945 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:32,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:33,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:37,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:37,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:42,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:43,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:43,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:45,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:45,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:45,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:46,209 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:47,209 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:47,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:51,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:52,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:53,431 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:48:58,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:58,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:58,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:58,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:03,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:08,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:08,340 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:08,340 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:08,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:09,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:09,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:09,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:13,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:13,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:14,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:19,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:23,432 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:49:25,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:28,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:28,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:28,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:28,803 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:28,803 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:28,804 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:29,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:30,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:31,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:33,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:35,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:40,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:43,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:43,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:45,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:45,351 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:45,352 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:45,354 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:46,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:46,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:47,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:49,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:51,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:53,433 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:49:57,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:58,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:58,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:58,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:02,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:05,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:05,630 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:05,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:05,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:06,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:07,240 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:07,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:11,241 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:12,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:13,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:18,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:23,434 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:50:24,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:27,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:27,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:27,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:27,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:28,247 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:29,248 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:29,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:29,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:29,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:30,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:31,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:33,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:35,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:40,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:44,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:46,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:47,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:47,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:47,778 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:47,778 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:48,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:49,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:51,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:53,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:53,434 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:50:57,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:59,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:59,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:59,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:03,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:08,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:11,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:11,189 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:11,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:11,190 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:11,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:13,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:14,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:14,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:14,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:14,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:15,268 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:19,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:23,435 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:51:24,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:29,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:29,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:30,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:30,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:30,379 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:30,379 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:30,379 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:31,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:31,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:35,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:35,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:40,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:44,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:44,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:45,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:45,524 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:45,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:45,525 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:46,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:46,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:47,285 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:51,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:51,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:53,436 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:51:57,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:59,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:59,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:59,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:00,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:00,796 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:00,796 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:00,797 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:01,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:02,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:03,291 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:05,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:08,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:13,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:14,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:14,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:14,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:15,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:15,101 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:15,101 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:15,102 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:15,299 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:17,300 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:18,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:19,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:23,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:23,437 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:52:28,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:29,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:34,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:36,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:36,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:36,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:36,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:36,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:37,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:39,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:41,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:44,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:44,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:44,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:49,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:51,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:54,356 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:54,356 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:54,359 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:54,359 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:52:54,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:55,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:55,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:59,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:59,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:59,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:00,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:05,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:10,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:12,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:12,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:12,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:12,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:12,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:13,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:14,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:14,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:14,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:16,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:17,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:21,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:23,439 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:53:26,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:29,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:29,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:29,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:31,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:35,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:35,475 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:35,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:35,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:36,330 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:37,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:37,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:39,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:42,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:44,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:44,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:44,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:47,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:50,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:50,459 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:50,460 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:50,460 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:51,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:51,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:53,440 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:53:53,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:55,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:58,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:59,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:59,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:59,644 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:03,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:07,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:07,353 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:07,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:07,356 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:08,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:09,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:09,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:11,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:14,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:14,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:14,629 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:14,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:19,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:22,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:22,647 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:22,647 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:22,648 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:23,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:23,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:23,441 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:54:25,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:27,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:29,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:29,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:29,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:30,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:35,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:40,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:44,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:44,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:44,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:44,556 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:44,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:44,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:44,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:45,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:45,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:45,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:49,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:50,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:53,442 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:54:56,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:59,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:59,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:59,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:01,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:02,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:02,729 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:02,729 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:02,730 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:03,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:03,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:06,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:07,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:11,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:14,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:14,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:14,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:17,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:20,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:20,139 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:20,140 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:20,141 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:20,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:21,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:23,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:23,443 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:55:25,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:28,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:29,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:29,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:29,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:33,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:38,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:42,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:42,914 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:42,914 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:42,915 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:43,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:43,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:43,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:44,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:44,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:44,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:47,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:49,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:53,444 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:55:54,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:59,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:59,626 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:59,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:00,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:00,713 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:00,713 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:00,713 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:01,384 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:01,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:04,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:05,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:09,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:14,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:14,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:14,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:15,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:19,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:19,145 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:19,145 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:19,146 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:19,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:19,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:21,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:23,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:23,445 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:56:26,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:29,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:29,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:29,845 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:31,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:36,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:36,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:36,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:36,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:36,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:37,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:37,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:41,399 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:42,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:44,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:44,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:44,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:47,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:52,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:53,446 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:56:56,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:56,842 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:56,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:56,843 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:57,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:57,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:58,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:59,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:59,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:59,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:01,406 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:03,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:08,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:13,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:14,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:14,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:14,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:19,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:21,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:57:21,105 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:57:21,106 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:57:21,106 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:57:21,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:57:21,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:23,447 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:57:25,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:25,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:29,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:29,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:29,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:30,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:35,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:38,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:57:38,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:57:38,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:57:38,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:57:38,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:57:39,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:41,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:41,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:44,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:44,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:44,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:46,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:51,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:53,452 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:57:57,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:59,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:59,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:59,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:01,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:01,265 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:01,265 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:01,265 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:01,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:01,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:03,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:05,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:08,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:13,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:14,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:14,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:14,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:18,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:18,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:18,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:18,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:18,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:19,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:19,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:23,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:23,449 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:58:24,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:29,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:29,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:35,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:35,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:35,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:35,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:35,719 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:36,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:37,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:39,444 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:40,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:44,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:44,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:44,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:45,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:50,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:52,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:52,924 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:52,925 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:52,925 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:53,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:53,450 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:58:54,450 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:55,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:58,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:59,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:59,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:59,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:01,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:06,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:09,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:09,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:09,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:09,555 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:10,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:12,457 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:12,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:14,457 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:14,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:14,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:14,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:17,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:22,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:23,451 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:59:24,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:24,289 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:24,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:24,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:24,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:26,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:28,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:28,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:29,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:33,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:38,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:43,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:44,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:44,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:44,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:44,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:44,928 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:44,928 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:44,931 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:45,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:46,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:49,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:50,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:53,452 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:59:54,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:59,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:59,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:59,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:59,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:03,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:03,227 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:03,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:03,227 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:03,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:04,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:05,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:08,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:10,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:14,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:14,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:14,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:15,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:20,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:23,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:23,045 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:23,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:23,048 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:23,453 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:00:23,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:24,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:26,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:28,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:29,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:31,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:36,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:41,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:43,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:43,247 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:43,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:43,248 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:43,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:44,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:44,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:47,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:48,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:52,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:53,455 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:00:57,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:59,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:59,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:59,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:00,640 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:00,640 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:00,641 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:01,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:02,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:02,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:06,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:07,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:12,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:14,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:17,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:17,328 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:17,328 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:17,329 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:18,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:18,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:22,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:23,456 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:01:23,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:28,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:29,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:33,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:34,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:34,859 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:34,859 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:34,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:35,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:36,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:39,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:40,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:44,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:44,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:45,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:50,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:53,457 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:01:55,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:59,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:59,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:59,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:01,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:01,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:01,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:01,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:01,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:02,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:04,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:06,525 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:06,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:11,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:14,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:14,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:17,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:22,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:23,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:23,123 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:23,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:23,124 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:23,457 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:02:23,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:24,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:27,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:28,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:29,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:29,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:33,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:38,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:42,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:42,062 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:42,063 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:42,063 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:42,538 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:43,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:44,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:44,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:44,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:44,982 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:46,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:48,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:53,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:53,459 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:02:58,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:59,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:59,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:59,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:02,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:02,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:02,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:02,180 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:02,546 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:04,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:04,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:06,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:08,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:09,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:14,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:14,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:14,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:14,968 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:19,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:20,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:20,561 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:20,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:20,562 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:21,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:22,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:23,460 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:03:25,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:26,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:29,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:29,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:29,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:31,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:36,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:38,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:38,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:38,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:38,291 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:38,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:40,560 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:41,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:42,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:44,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:44,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:44,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:44,996 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:47,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:52,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:53,460 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:03:57,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:58,059 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:58,059 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:58,060 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:58,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:00,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:00,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:00,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:00,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:03,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:08,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:13,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:15,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:15,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:15,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:15,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:15,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:15,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:16,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:18,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:20,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:23,461 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:04:23,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:28,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:30,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:30,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:33,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:38,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:41,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:41,062 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:41,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:41,063 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:41,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:42,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:44,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:45,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:45,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:45,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:46,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:49,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:53,462 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:04:55,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:55,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:55,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:55,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:55,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:56,589 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:58,590 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:00,591 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:00,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:00,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:00,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:00,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:05,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:10,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:15,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:15,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:15,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:15,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:17,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:05:17,261 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:05:17,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:05:17,261 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:05:17,597 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:05:20,598 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:21,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:22,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:23,463 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:05:27,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:30,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:30,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:32,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:37,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:41,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:05:41,340 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:05:41,340 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:05:41,341 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:05:41,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:05:42,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:43,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:45,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:45,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:45,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:46,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:48,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:53,464 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:05:54,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:59,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:00,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:00,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:00,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:00,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:00,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:00,656 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:02,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:05,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:06,619 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:10,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:15,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:15,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:15,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:15,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:20,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:22,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:22,657 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:22,657 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:22,657 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:23,465 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:06:23,625 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:24,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:25,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:28,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:30,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:30,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:36,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:39,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:39,451 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:39,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:39,452 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:39,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:40,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:42,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:44,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:45,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:45,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:45,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:47,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:52,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:53,467 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:06:57,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:57,310 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:57,310 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:57,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:57,639 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:58,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:58,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:00,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:00,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:00,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:02,648 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:03,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:08,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:13,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:15,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:15,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:15,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:16,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:16,886 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:16,886 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:16,886 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:17,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:18,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:18,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:22,658 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:23,467 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:07:24,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:29,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:30,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:30,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:30,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:34,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:39,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:40,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:40,780 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:40,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:40,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:41,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:42,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:44,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:45,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:45,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:45,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:46,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:50,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:53,469 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:07:56,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:57,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:57,405 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:57,406 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:57,406 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:57,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:58,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:00,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:00,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:01,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:02,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:06,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:11,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:12,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:12,997 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:12,997 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:12,997 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:13,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:14,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:15,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:15,713 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:15,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:17,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:18,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:22,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:23,469 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:08:27,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:29,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:29,908 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:29,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:29,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:30,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:30,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:32,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:32,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:34,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:38,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:43,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:45,659 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:45,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:48,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:48,006 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:48,006 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:48,007 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:48,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:49,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:50,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:52,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:53,470 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:08:54,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:59,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:00,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:00,659 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:04,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:08,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:08,475 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:08,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:08,476 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:08,698 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:10,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:10,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:12,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:15,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:15,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:15,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:20,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:23,471 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:09:26,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:26,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:26,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:26,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:26,347 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:26,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:28,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:30,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:30,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:30,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:31,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:36,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:41,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:45,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:46,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:46,700 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:46,700 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:46,701 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:46,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:46,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:48,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:50,714 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:51,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:53,472 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:09:57,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:00,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:00,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:00,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:00,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:00,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:00,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:01,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:02,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:02,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:04,719 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:06,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:08,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:13,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:15,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:18,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:19,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:19,268 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:19,268 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:19,269 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:19,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:20,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:23,473 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:10:24,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:24,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:29,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:30,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:30,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:34,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:39,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:41,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:41,161 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:41,162 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:41,162 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:41,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:42,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:45,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:45,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:45,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:46,736 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:50,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:53,474 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:10:56,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:00,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:01,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:02,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:02,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:02,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:02,519 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:02,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:04,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:06,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:07,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:12,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:15,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:17,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:19,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:19,261 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:19,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:19,262 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:19,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:20,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:23,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:23,475 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:11:24,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:28,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:30,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:33,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:38,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:39,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:39,141 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:39,142 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:39,142 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:39,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:40,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:44,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:44,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:45,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:49,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:53,476 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:11:55,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:57,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:57,867 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:57,867 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:57,867 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:58,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:58,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:00,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:02,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:06,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:11,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:15,184 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:15,185 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:15,186 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:15,186 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:15,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:15,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:15,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:16,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:16,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:20,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:22,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:23,477 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:12:27,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:30,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:32,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:32,167 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:32,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:32,168 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:32,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:33,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:34,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:36,779 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:38,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:43,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:45,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:45,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:48,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:48,522 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:48,522 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:48,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:48,523 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:48,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:50,784 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:52,785 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:53,478 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:12:54,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:59,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:00,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:03,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:03,256 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:03,256 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:03,257 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:03,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:04,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:05,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:08,791 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:11,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:15,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:15,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:15,665 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:16,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:21,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:23,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:23,206 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:23,206 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:23,207 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:23,479 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:13:23,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:24,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:27,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:28,798 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:30,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:30,665 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:32,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:37,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:42,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:42,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:42,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:42,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:42,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:43,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:44,804 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:45,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:45,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:46,805 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:48,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:53,480 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:13:54,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:59,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:00,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:00,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:04,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:06,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:06,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:06,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:06,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:06,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:06,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:10,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:10,813 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:15,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:15,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:15,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:20,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:20,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:20,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:20,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:20,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:20,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:22,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:23,481 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:14:24,819 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:25,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:30,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:30,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:31,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:36,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:39,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:39,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:39,347 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:39,347 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:39,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:40,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:42,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:44,830 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:45,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:45,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:47,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:52,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:53,482 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:14:55,199 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:55,200 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:55,201 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:55,201 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:55,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:56,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:58,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:58,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:00,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:00,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:03,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:08,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:11,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:11,516 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:11,516 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:11,517 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:11,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:12,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:14,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:15,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:15,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:15,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:16,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:19,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:23,483 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:15:25,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:26,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:26,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:26,344 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:26,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:26,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:26,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:30,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:30,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:30,710 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:30,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:35,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:40,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:45,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:45,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:46,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:51,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:52,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:52,330 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:52,330 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:52,331 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:52,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:52,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:53,483 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:15:56,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:57,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:00,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:00,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:00,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:02,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:07,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:10,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:10,182 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:10,182 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:10,183 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:10,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:10,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:13,197 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:14,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:15,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:15,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:15,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:18,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:23,484 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:16:24,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:29,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:30,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:30,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:30,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:32,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:32,689 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:32,690 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:32,690 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:32,873 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:34,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:34,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:36,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:39,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:44,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:45,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:45,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:45,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:50,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:50,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:50,392 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:50,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:50,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:50,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:51,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:53,485 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:16:55,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:55,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:00,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:00,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:00,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:00,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:04,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:04,150 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:04,150 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:04,151 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:04,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:05,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:06,184 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:09,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:11,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:15,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:15,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:15,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:16,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:21,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:22,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:22,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:22,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:22,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:22,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:23,486 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:17:25,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:27,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:27,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:30,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:30,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:30,676 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:32,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:37,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:40,173 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:40,175 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:40,175 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:40,176 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:40,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:41,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:43,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:45,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:45,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:45,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:45,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:48,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:53,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:54,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:56,067 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:17:56,068 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:56,068 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:56,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:56,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:57,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:59,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:59,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:00,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:00,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:00,681 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:04,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:09,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:13,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:13,746 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:13,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:13,747 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:13,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:15,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:15,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:16,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:16,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:16,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:19,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:20,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:23,488 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:18:26,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:31,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:31,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:31,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:31,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:36,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:36,058 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:36,058 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:36,060 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:36,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:37,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:37,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:41,926 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:42,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:46,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:46,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:46,386 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:48,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:53,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:53,489 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:18:53,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:53,666 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:53,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:53,667 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:53,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:55,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:58,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:59,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:01,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:01,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:04,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:09,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:10,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:10,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:10,154 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:10,156 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:10,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:11,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:15,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:15,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:16,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:16,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:21,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:23,490 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:19:23,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:23,792 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:23,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:23,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:23,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:25,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:26,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:29,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:31,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:31,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:31,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:32,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:41,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:41,198 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:41,198 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:41,200 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:41,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:43,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:43,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:45,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:46,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:46,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:48,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:53,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:53,490 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:19:58,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:59,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:59,599 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:59,599 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:59,599 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:59,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:01,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:01,959 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:03,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:03,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:05,961 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:08,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:13,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:16,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:18,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:18,096 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:18,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:18,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:18,965 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:19,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:19,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:23,491 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:20:23,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:24,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:29,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:31,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:31,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:31,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:32,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:32,902 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:32,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:32,902 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:32,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:34,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:35,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:37,973 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:40,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:45,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:46,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:49,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:49,397 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:49,397 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:49,399 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:49,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:50,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:51,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:53,492 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:20:53,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:55,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:00,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:01,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:06,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:11,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:13,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:21:13,832 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:21:13,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:21:13,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:21:13,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:21:15,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:16,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:16,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:17,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:19,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:22,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:23,493 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:21:27,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:31,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:31,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:31,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:32,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:33,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:21:33,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:21:33,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:21:33,782 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:21:33,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:21:35,996 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:37,996 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:38,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:39,997 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:43,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:46,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:46,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:46,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:48,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:53,494 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:21:54,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:59,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:01,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:01,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:01,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:01,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:01,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:01,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:01,766 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:02,005 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:04,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:04,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:06,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:08,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:10,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:15,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:16,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:16,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:16,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:20,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:23,495 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:22:23,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:23,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:23,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:23,829 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:24,013 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:25,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:26,014 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:28,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:30,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:31,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:31,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:31,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:36,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:40,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:40,400 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:40,401 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:40,401 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:41,020 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:42,020 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:42,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:46,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:46,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:46,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:46,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:47,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:52,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:53,502 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:22:58,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:01,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:01,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:01,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:03,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:05,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:05,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:05,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:05,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:06,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:08,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:08,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:10,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:12,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:14,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:16,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:16,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:19,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:21,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:21,266 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:21,266 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:21,267 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:22,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:23,498 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:23:24,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:24,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:26,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:29,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:31,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:31,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:31,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:35,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:40,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:44,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:44,775 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:44,775 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:44,776 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:45,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:46,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:46,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:46,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:50,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:51,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:53,498 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:23:56,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:01,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:01,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:01,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:01,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:03,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:03,228 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:03,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:03,230 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:04,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:06,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:07,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:08,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:12,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:16,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:16,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:16,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:17,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:22,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:23,500 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:24:25,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:25,305 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:25,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:25,306 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:26,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:28,060 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:28,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:30,061 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:31,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:33,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:38,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:42,024 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:42,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:42,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:42,027 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:42,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:44,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:44,066 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:46,067 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:46,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:49,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:53,500 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:24:55,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:00,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:01,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:01,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:01,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:01,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:01,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:02,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:04,074 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:06,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:06,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:11,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:16,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:16,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:16,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:16,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:21,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:23,502 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:25:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:25,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:25,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:25,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:26,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:27,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:28,082 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:30,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:31,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:32,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:37,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:42,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:46,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:46,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:46,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:48,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:48,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:48,793 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:48,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:48,794 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:49,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:50,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:53,503 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:25:54,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:54,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:59,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:01,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:01,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:04,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:09,154 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:09,155 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:09,155 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:09,155 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:10,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:10,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:10,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:14,099 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:15,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:16,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:16,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:16,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:20,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:23,504 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:26:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:25,237 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:25,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:25,238 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:26,104 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:26,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:26,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:30,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:31,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:31,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:36,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:41,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:46,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:46,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:46,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:47,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:51,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:51,320 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:51,321 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:51,321 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:52,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:52,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:53,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:53,505 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:26:56,116 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:58,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:01,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:01,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:01,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:03,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:07,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:07,608 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:07,609 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:07,609 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:08,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:08,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:10,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:12,123 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:13,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:16,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:16,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:16,508 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:19,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:23,505 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:27:25,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:29,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:29,472 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:29,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:29,473 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:30,130 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:30,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:32,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:32,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:32,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:32,131 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:34,132 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:35,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:40,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:44,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:44,847 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:44,847 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:44,848 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:45,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:45,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:46,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:47,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:47,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:47,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:50,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:51,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:53,507 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:27:56,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:01,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:02,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:02,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:02,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:02,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:02,831 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:02,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:02,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:03,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:04,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:07,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:08,144 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:12,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:15,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:15,883 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:15,883 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:15,883 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:16,148 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:17,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:17,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:18,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:20,150 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:23,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:23,508 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:28:28,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:32,587 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:33,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:37,470 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:37,470 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:37,471 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:38,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:38,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:39,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:42,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:44,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:47,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:50,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:53,509 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:28:56,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:01,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:02,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:02,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:02,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:06,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:06,108 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:06,108 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:06,109 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:06,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:07,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:08,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:10,169 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:12,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:17,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:17,587 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:17,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:22,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:23,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:29:24,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:24,294 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:24,295 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:24,295 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:25,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:26,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:28,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:28,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:32,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:32,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:33,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:38,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:38,252 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:38,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:38,252 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:39,179 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:39,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:40,180 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:42,180 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:44,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:47,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:47,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:49,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:53,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:29:55,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:55,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:55,932 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:55,932 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:55,933 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:56,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:58,186 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:00,187 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:01,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:02,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:02,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:02,589 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:06,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:11,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:11,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:11,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:11,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:12,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:12,192 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:12,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:16,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:17,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:17,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:17,622 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:17,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:22,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:23,512 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:30:26,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:26,878 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:26,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:26,878 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:27,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:27,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:28,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:32,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:32,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:32,593 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:32,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:33,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:38,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:43,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:45,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:45,579 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:45,579 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:45,579 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:46,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:46,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:48,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:48,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:48,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:48,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:50,207 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:53,513 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:30:54,514 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:59,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:03,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:03,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:03,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:04,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:04,260 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:04,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:05,253 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:05,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:06,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:06,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:08,214 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:10,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:15,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:18,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:18,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:18,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:19,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:19,033 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:19,033 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:19,033 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:19,218 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:20,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:21,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:23,514 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:31:24,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:26,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:31,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:33,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:33,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:33,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:36,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:36,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:36,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:36,271 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:37,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:37,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:38,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:40,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:42,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:47,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:48,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:48,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:48,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:53,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:53,514 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:31:53,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:53,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:53,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:53,532 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:54,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:54,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:58,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:58,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:03,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:03,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:03,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:04,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:09,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:10,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:10,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:10,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:10,689 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:11,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:12,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:14,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:14,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:16,240 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:18,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:18,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:18,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:20,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:23,515 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:32:25,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:28,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:28,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:28,654 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:28,654 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:29,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:30,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:30,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:32,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:33,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:33,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:33,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:36,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:41,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:46,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:46,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:46,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:46,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:46,831 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:47,254 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:48,254 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:48,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:48,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:48,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:50,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:51,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:52,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:54,707 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:32:57,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:02,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:03,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:03,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:03,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:08,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:10,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:10,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:10,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:10,499 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:11,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:12,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:13,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:14,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:18,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:18,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:18,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:19,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:23,517 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:33:25,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:30,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:30,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:30,927 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:30,928 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:30,929 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:31,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:32,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:33,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:33,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:33,657 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:36,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:36,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:41,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:46,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:47,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:47,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:47,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:47,519 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:48,277 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:48,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:49,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:51,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:52,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:53,518 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:33:57,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:02,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:04,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:04,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:04,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:06,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:06,499 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:06,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:06,501 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:07,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:07,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:08,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:10,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:12,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:17,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:19,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:19,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:19,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:23,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:23,519 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:34:25,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:25,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:25,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:25,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:25,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:27,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:29,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:31,298 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:34,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:34,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:34,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:34,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:39,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:44,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:48,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:48,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:48,366 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:48,368 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:49,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:49,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:49,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:49,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:51,306 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:53,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:53,520 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:34:54,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:59,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:04,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:04,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:05,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:05,240 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:05,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:05,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:05,241 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:05,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:07,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:10,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:11,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:15,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:19,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:19,226 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:19,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:21,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:23,521 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:35:25,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:25,140 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:25,140 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:25,141 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:25,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:27,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:27,320 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:31,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:32,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:34,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:34,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:34,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:38,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:40,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:40,691 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:40,691 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:40,691 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:41,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:43,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:43,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:45,327 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:49,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:49,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:49,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:49,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:53,522 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:35:54,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:57,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:57,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:57,601 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:57,601 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:58,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:59,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:59,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:03,334 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:04,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:04,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:04,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:05,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:10,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:15,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:19,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:19,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:19,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:20,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:20,806 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:20,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:20,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:20,807 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:21,341 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:23,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:23,523 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:36:25,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:26,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:31,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:34,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:34,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:34,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:36,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:39,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:39,040 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:39,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:39,040 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:39,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:41,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:42,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:43,350 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:47,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:49,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:49,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:49,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:52,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:53,524 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:36:54,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:54,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:54,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:54,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:55,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:57,355 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:57,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:59,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:02,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:04,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:04,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:04,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:08,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:13,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:14,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:14,224 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:14,224 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:14,225 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:14,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:15,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:18,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:19,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:19,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:19,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:19,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:23,525 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:37:24,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:29,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:31,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:31,591 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:31,591 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:31,592 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:32,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:33,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:34,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:34,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:34,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:34,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:37,372 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:39,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:44,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:49,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:50,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:53,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:53,493 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:53,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:53,494 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:53,526 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:37:54,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:55,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:55,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:59,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:00,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:04,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:06,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:11,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:14,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:14,188 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:14,188 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:14,188 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:38:14,386 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:38:15,386 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:19,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:19,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:19,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:19,388 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:22,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:23,527 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:38:27,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:32,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:32,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:32,989 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:32,990 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:32,990 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:38:33,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:38:34,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:34,722 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:34,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:35,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:37,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:38,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:39,396 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:43,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:48,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:49,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:49,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:49,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:53,528 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:38:53,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:58,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:59,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:59,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:59,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:59,822 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:00,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:01,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:03,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:04,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:04,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:04,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:05,407 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:09,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:14,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:19,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:19,258 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:19,258 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:19,259 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:19,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:19,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:19,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:19,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:20,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:21,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:23,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:23,529 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:39:26,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:31,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:34,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:36,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:41,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:41,564 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:41,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:41,564 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:42,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:42,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:43,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:47,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:47,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:49,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:49,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:52,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:53,529 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:39:58,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:59,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:59,055 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:59,055 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:59,056 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:59,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:01,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:03,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:04,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:04,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:04,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:09,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:14,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:19,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:19,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:19,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:20,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:23,531 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:40:25,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:27,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:40:27,345 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:40:27,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:40:27,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:40:27,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:29,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:31,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:31,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:34,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:34,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:34,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:36,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:45,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:40:45,361 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:40:45,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:40:45,361 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:40:45,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:47,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:47,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:49,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:49,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:49,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:49,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:52,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:53,532 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:40:58,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:02,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:02,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:02,070 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:02,070 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:02,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:03,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:04,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:04,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:04,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:07,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:09,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:14,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:17,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:17,239 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:17,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:17,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:17,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:19,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:19,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:19,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:19,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:19,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:21,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:23,533 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:41:24,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:29,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:34,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:34,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:34,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:39,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:43,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:43,371 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:43,372 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:43,372 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:43,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:45,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:45,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:47,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:49,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:49,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:50,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:53,534 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:41:56,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:01,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:01,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:01,788 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:01,789 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:01,789 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:02,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:03,478 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:04,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:04,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:04,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:06,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:07,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:11,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:16,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:18,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:18,888 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:18,888 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:18,888 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:19,484 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:20,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:20,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:20,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:21,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:22,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:23,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:23,534 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:42:27,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:32,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:35,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:35,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:35,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:37,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:39,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:39,341 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:39,341 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:39,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:39,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:41,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:43,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:43,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:48,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:50,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:50,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:50,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:53,535 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:42:54,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:56,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:56,373 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:56,373 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:56,373 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:56,498 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:57,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:00,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:01,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:05,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:05,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:05,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:05,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:10,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:15,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:16,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:16,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:16,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:16,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:17,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:20,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:20,685 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:20,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:20,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:21,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:23,536 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:43:26,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:31,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:31,190 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:31,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:31,211 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:31,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:32,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:33,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:35,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:35,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:35,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:35,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:37,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:42,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:47,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:50,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:50,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:50,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:51,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:51,408 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:51,408 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:51,409 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:51,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:53,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:53,520 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:53,537 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:43:55,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:58,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:03,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:05,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:05,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:05,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:08,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:11,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:11,855 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:11,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:11,856 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:12,527 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:13,528 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:13,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:15,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:17,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:18,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:20,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:20,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:20,675 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:23,538 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:44:24,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:26,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:26,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:26,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:26,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:27,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:29,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:29,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:31,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:35,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:35,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:35,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:35,716 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:40,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:44,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:44,506 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:44,506 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:44,507 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:44,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:45,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:46,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:49,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:50,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:50,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:50,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:51,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:53,539 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:44:57,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:01,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:01,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:01,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:01,718 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:02,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:02,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:03,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:05,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:05,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:05,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:05,689 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:08,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:13,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:18,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:20,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:20,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:20,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:21,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:21,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:21,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:21,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:21,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:23,540 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:45:23,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:24,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:25,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:29,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:34,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:35,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:35,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:35,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:37,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:37,897 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:37,898 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:37,898 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:38,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:39,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:39,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:41,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:43,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:45,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:50,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:50,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:50,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:50,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:52,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:52,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:52,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:52,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:53,541 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:45:53,566 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:53,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:55,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:57,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:00,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:05,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:05,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:05,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:06,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:11,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:16,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:18,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:46:18,890 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:46:18,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:46:18,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:46:19,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:46:19,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:21,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:21,482 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:21,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:21,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:23,542 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:46:23,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:27,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:32,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:36,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:36,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:36,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:37,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:42,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:45,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:46:45,939 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:46:45,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:46:45,940 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:46:46,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:46:47,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:47,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:49,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:51,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:51,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:51,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:53,543 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:46:53,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:58,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:02,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:02,843 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:02,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:02,843 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:03,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:03,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:03,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:06,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:06,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:06,500 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:07,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:09,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:14,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:19,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:21,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:21,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:21,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:23,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:23,360 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:23,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:23,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:23,544 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:47:23,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:25,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:25,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:27,602 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:30,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:36,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:36,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:36,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:36,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:41,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:43,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:43,806 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:43,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:43,807 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:44,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:45,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:46,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:47,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:51,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:51,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:51,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:52,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:53,545 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:47:58,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:01,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:01,122 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:01,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:01,123 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:01,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:01,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:05,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:06,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:06,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:06,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:09,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:14,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:16,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:16,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:16,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:16,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:17,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:19,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:21,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:21,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:21,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:21,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:23,546 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:48:25,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:30,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:33,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:33,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:33,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:33,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:34,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:35,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:35,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:36,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:36,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:37,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:40,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:45,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:50,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:51,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:51,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:51,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:53,547 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:48:53,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:53,580 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:53,580 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:53,581 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:53,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:55,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:56,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:57,636 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:01,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:06,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:06,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:06,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:07,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:12,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:14,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:14,706 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:14,706 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:14,706 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:17,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:19,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:21,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:21,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:21,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:23,548 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:49:23,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:28,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:33,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:36,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:36,278 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:36,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:36,279 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:36,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:36,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:36,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:36,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:37,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:38,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:39,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:41,653 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:44,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:49,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:51,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:51,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:51,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:53,549 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:49:54,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:55,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:55,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:55,440 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:55,440 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:55,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:57,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:59,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:59,661 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:04,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:06,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:06,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:06,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:10,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:15,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:15,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:15,745 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:15,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:15,746 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:16,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:17,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:19,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:20,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:21,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:21,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:21,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:23,550 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:50:26,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:31,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:36,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:36,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:38,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:38,106 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:38,106 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:38,107 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:38,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:39,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:41,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:42,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:47,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:51,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:51,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:51,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:52,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:53,551 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:50:57,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:57,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:57,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:57,301 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:57,681 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:57,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:58,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:01,683 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:03,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:06,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:06,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:06,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:08,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:11,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:11,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:11,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:11,165 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:11,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:11,687 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:14,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:15,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:19,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:21,550 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:23,552 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:51:24,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:29,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:32,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:32,677 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:32,678 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:32,678 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:32,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:33,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:34,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:36,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:36,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:36,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:37,696 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:40,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:45,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:47,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:47,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:47,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:47,499 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:47,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:47,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:50,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:51,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:51,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:51,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:51,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:53,553 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:51:56,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:01,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:03,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:03,137 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:03,137 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:03,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:03,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:03,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:06,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:06,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:06,549 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:06,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:07,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:11,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:16,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:17,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:17,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:17,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:17,313 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:17,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:17,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:21,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:21,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:21,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:22,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:22,714 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:23,554 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:52:28,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:33,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:33,452 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:33,453 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:33,453 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:33,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:34,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:34,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:36,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:36,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:36,549 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:38,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:39,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:44,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:49,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:49,311 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:49,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:49,312 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:49,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:50,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:50,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:51,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:51,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:51,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:53,555 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:52:54,726 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:55,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:00,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:05,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:06,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:06,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:06,915 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:08,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:08,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:08,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:08,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:09,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:10,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:10,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:14,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:16,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:21,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:21,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:21,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:21,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:23,556 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:53:26,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:26,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:26,843 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:26,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:26,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:27,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:28,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:31,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:32,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:36,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:36,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:36,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:37,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:40,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:40,035 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:40,036 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:40,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:40,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:42,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:42,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:44,744 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:47,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:51,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:51,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:51,917 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:52,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:53,557 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:53:58,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:00,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:00,289 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:00,289 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:00,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:00,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:02,751 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:03,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:04,752 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:06,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:06,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:06,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:09,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:14,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:19,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:20,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:20,638 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:20,639 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:20,639 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:20,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:22,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:22,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:22,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:22,759 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:23,558 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:54:24,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:24,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:26,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:34,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:34,614 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:34,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:34,615 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:34,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:35,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:36,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:37,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:37,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:37,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:38,765 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:40,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:45,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:50,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:51,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:51,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:51,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:51,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:51,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:52,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:52,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:52,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:53,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:53,559 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:54:56,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:56,772 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:06,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:07,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:07,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:07,365 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:07,365 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:07,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:07,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:07,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:08,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:08,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:11,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:12,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:16,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:22,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:22,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:23,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:23,559 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:55:26,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:27,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:27,391 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:27,391 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:27,391 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:27,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:28,784 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:31,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:32,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:36,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:37,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:37,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:38,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:42,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:43,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:43,374 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:43,375 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:43,375 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:43,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:44,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:47,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:48,792 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:52,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:52,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:52,998 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:53,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:53,561 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:55:57,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:02,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:04,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:04,809 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:04,809 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:04,809 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:04,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:06,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:07,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:07,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:08,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:08,001 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:08,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:10,819 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:13,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:18,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:22,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:22,028 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:22,028 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:22,029 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:22,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:23,562 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:56:23,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:23,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:23,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:23,771 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:24,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:26,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:29,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:34,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:38,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:38,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:38,770 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:39,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:39,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:39,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:39,180 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:39,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:40,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:44,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:45,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:50,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:53,562 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:56:53,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:53,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:53,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:55,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:58,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:58,497 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:58,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:58,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:58,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:00,841 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:01,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:02,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:06,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:08,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:08,776 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:08,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:12,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:17,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:22,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:22,136 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:22,136 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:22,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:22,850 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:23,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:23,563 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:57:23,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:23,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:23,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:24,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:26,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:28,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:33,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:38,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:38,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:38,789 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:38,789 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:38,790 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:38,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:38,877 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:38,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:38,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:40,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:44,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:44,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:49,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:53,564 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:57:53,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:53,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:54,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:54,360 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:54,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:54,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:54,361 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:54,864 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:58,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:59,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:04,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:08,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:08,883 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:08,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:09,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:09,574 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:09,575 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:09,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:09,575 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:09,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:10,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:14,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:14,872 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:19,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:23,566 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:58:23,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:23,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:23,920 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:25,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:29,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:29,156 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:29,156 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:29,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:29,878 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:30,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:30,878 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:34,880 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:35,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:38,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:38,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:38,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:41,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:46,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:50,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:50,442 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:50,442 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:50,442 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:50,886 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:51,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:52,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:53,566 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:58:53,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:53,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:53,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:54,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:57,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:02,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:05,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:05,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:05,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:05,857 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:05,892 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:06,893 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:07,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:08,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:10,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:13,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:18,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:23,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:23,568 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:59:23,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:23,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:23,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:24,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:24,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:24,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:24,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:24,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:26,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:28,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:28,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:33,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:38,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:38,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:38,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:44,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:48,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:48,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:48,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:48,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:48,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:48,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:49,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:52,911 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:53,569 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:59:53,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:53,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:54,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:59,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:01,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:01,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:01,798 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:02,827 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:02,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:04,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:05,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:06,916 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:08,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:10,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:15,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:20,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:23,570 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:00:23,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:23,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:23,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:25,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:25,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:25,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:25,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:25,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:26,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:26,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:28,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:31,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:36,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:38,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:38,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:41,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:42,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:42,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:42,315 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:42,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:42,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:44,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:46,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:47,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:52,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:53,571 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:00:53,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:53,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:57,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:02,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:02,504 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:02,504 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:02,505 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:02,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:03,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:04,939 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:06,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:08,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:08,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:13,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:18,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:20,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:20,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:20,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:20,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:20,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:22,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:23,571 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:01:23,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:23,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:23,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:23,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:24,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:28,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:33,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:38,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:38,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:38,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:42,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:42,559 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:42,559 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:42,559 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:42,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:44,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:44,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:46,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:49,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:53,572 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:01:53,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:53,893 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:53,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:54,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:59,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:01,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:01,889 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:01,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:01,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:01,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:02,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:04,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:06,962 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:08,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:08,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:08,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:09,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:14,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:19,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:23,573 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:02:23,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:23,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:23,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:24,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:24,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:24,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:24,518 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:24,968 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:25,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:26,969 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:28,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:30,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:35,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:38,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:38,893 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:38,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:40,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:43,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:43,724 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:43,724 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:43,725 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:43,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:44,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:46,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:48,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:52,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:53,574 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:02:53,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:53,937 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:53,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:57,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:57,425 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:57,426 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:57,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:57,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:57,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:00,982 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:02,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:02,983 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:07,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:08,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:08,897 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:08,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:13,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:18,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:20,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:03:20,466 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:03:20,466 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:03:20,466 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:03:20,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:20,990 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:23,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:23,575 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:03:23,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:23,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:23,941 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:24,991 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:29,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:34,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:38,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:38,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:38,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:39,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:44,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:03:44,050 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:03:44,050 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:03:44,051 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:03:44,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:44,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:45,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:49,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:50,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:53,576 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:03:53,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:53,926 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:53,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:56,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:01,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:03,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:03,993 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:03,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:03,993 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:04,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:05,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:06,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:08,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:08,907 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:08,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:09,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:12,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:17,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:22,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:23,577 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:04:23,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:23,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:23,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:25,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:25,436 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:25,436 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:25,437 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:26,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:27,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:27,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:31,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:32,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:37,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:38,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:38,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:38,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:42,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:42,609 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:42,609 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:42,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:42,610 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:43,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:45,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:47,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:47,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:52,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:53,577 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:04:53,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:53,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:53,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:58,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:58,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:58,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:58,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:58,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:59,028 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:01,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:03,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:03,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:08,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:08,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:08,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:08,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:14,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:19,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:20,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:20,889 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:20,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:20,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:21,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:23,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:23,579 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:05:23,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:23,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:23,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:24,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:25,039 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:29,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:34,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:37,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:37,257 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:37,257 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:37,257 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:38,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:38,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:38,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:38,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:39,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:40,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:43,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:45,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:50,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:53,580 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:05:53,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:53,982 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:53,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:54,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:54,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:54,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:54,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:55,050 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:56,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:57,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:59,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:01,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:06,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:08,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:08,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:09,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:11,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:11,092 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:11,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:11,093 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:12,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:12,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:13,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:17,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:17,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:22,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:23,581 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:06:23,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:23,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:23,990 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:25,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:25,550 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:25,551 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:25,551 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:26,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:27,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:28,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:31,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:33,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:38,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:38,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:38,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:38,989 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:44,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:46,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:46,810 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:46,810 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:46,810 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:47,069 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:49,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:49,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:51,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:53,581 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:06:53,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:53,989 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:53,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:55,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:00,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:02,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:02,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:02,528 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:02,806 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:03,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:05,076 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:05,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:07,076 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:09,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:09,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:09,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:11,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:16,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:17,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:17,853 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:17,853 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:17,853 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:18,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:19,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:21,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:23,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:23,582 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:07:23,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:23,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:24,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:27,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:32,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:37,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:38,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:38,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:39,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:42,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:42,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:42,819 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:42,820 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:42,820 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:43,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:45,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:47,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:48,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:53,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:53,583 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:07:53,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:54,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:54,001 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:58,129 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:59,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:59,170 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:59,170 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:59,171 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:00,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:03,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:03,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:08,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:08,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:08,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:09,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:14,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:19,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:21,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:21,973 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:21,973 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:21,974 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:22,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:23,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:23,585 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:08:24,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:24,571 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:24,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:24,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:27,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:30,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:35,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:38,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:38,490 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:38,490 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:38,490 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:39,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:39,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:40,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:40,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:40,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:40,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:43,113 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:45,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:50,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:53,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:53,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:53,513 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:53,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:53,586 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:08:54,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:55,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:55,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:55,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:55,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:56,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:57,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:59,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:01,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:06,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:10,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:12,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:14,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:14,756 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:14,756 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:14,757 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:09:15,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:09:17,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:17,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:19,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:22,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:23,587 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:09:25,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:25,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:25,232 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:28,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:33,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:35,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:35,646 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:35,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:35,646 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:09:36,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:09:37,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:38,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:40,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:40,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:40,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:41,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:44,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:49,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:53,588 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:09:54,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:55,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:55,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:55,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:59,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:59,547 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:59,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:59,548 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:00,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:00,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:01,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:03,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:05,147 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:05,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:10,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:10,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:10,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:11,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:16,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:21,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:23,588 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:10:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:25,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:26,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:10:26,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:10:26,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:10:26,270 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:27,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:27,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:27,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:31,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:32,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:37,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:40,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:40,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:43,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:45,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:10:45,375 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:10:45,375 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:10:45,375 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:46,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:47,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:48,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:49,169 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:53,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:53,589 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:10:55,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:55,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:55,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:59,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:03,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:03,047 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:03,047 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:03,048 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:03,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:05,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:06,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:08,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:10,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:10,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:10,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:10,237 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:15,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:20,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:23,591 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:11:25,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:25,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:25,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:25,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:27,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:27,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:27,514 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:27,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:28,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:30,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:30,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:32,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:35,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:40,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:40,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:41,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:42,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:42,835 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:42,835 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:42,835 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:43,189 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:46,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:47,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:48,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:52,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:53,592 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:11:55,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:55,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:55,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:57,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:59,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:59,143 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:59,143 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:59,153 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:59,195 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:02,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:02,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:04,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:07,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:10,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:10,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:10,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:13,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:17,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:12:17,611 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:12:17,612 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:12:17,612 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:12:18,202 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:18,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:20,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:22,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:23,593 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:12:24,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:25,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:25,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:25,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:30,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:35,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:40,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:40,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:41,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:41,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:12:41,523 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:12:41,524 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:12:41,524 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:12:42,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:44,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:46,214 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:46,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:51,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:53,594 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:12:55,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:55,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:55,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:57,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:01,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:01,753 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:01,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:01,754 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:02,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:02,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:04,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:06,222 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:07,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:10,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:10,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:10,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:13,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:18,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:19,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:19,068 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:19,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:19,069 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:19,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:22,227 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:23,595 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:13:23,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:24,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:25,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:25,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:25,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:29,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:34,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:37,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:37,339 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:37,339 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:37,339 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:38,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:39,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:40,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:40,257 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:40,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:42,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:44,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:49,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:53,596 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:13:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:55,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:55,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:55,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:00,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:02,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:02,087 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:02,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:02,088 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:02,243 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:04,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:05,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:08,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:10,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:10,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:10,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:11,294 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:16,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:21,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:21,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:21,831 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:21,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:21,831 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:22,252 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:23,597 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:14:24,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:25,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:25,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:25,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:26,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:26,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:31,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:36,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:36,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:36,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:36,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:37,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:37,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:38,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:40,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:40,250 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:40,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:42,260 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:42,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:47,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:52,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:53,597 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:14:55,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:55,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:55,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:58,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:58,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:58,664 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:58,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:58,664 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:59,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:00,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:03,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:04,268 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:08,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:10,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:10,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:10,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:14,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:19,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:21,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:21,121 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:21,122 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:21,122 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:21,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:23,598 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:15:24,275 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:25,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:25,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:25,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:25,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:26,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:30,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:35,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:35,080 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:35,080 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:35,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:35,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:36,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:36,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:40,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:40,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:40,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:40,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:41,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:46,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:51,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:53,599 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:15:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:55,269 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:55,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:55,733 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:55,733 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:55,733 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:56,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:56,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:58,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:00,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:01,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:06,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:10,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:10,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:12,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:16,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:16:16,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:16:16,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:16:16,547 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:16:17,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:16:17,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:18,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:22,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:22,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:23,599 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:16:25,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:25,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:25,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:28,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:33,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:38,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:40,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:40,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:40,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:42,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:16:42,525 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:16:42,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:16:42,525 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:16:43,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:16:43,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:44,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:48,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:48,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:53,600 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:16:54,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:55,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:55,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:55,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:00,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:00,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:00,468 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:00,468 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:00,469 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:01,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:02,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:05,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:06,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:10,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:10,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:10,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:11,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:16,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:18,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:18,299 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:18,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:18,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:18,318 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:20,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:21,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:23,601 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:17:24,320 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:25,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:25,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:25,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:26,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:31,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:35,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:35,981 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:35,981 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:35,982 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:36,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:37,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:38,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:40,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:40,272 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:40,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:40,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:42,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:47,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:52,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:53,480 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:53,481 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:53,481 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:53,482 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:53,602 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:17:54,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:56,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:56,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:56,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:56,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:57,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:58,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:02,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:07,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:11,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:11,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:11,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:13,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:16,129 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:16,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:16,131 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:16,131 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:16,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:18,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:18,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:20,340 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:23,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:23,608 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:18:26,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:26,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:26,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:29,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:34,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:34,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:34,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:34,086 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:34,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:35,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:36,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:38,346 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:40,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:41,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:46,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:51,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:51,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:51,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:51,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:51,668 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:52,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:53,605 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:18:54,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:56,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:56,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:56,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:01,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:06,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:08,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:08,171 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:08,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:08,172 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:08,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:10,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:11,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:11,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:11,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:11,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:12,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:16,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:21,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:23,606 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:19:26,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:26,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:26,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:27,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:29,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:29,248 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:29,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:29,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:29,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:30,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:32,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:34,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:37,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:41,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:41,092 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:43,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:45,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:45,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:45,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:45,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:45,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:46,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:48,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:50,375 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:53,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:53,606 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:19:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:56,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:56,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:59,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:04,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:07,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:07,482 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:07,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:07,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:08,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:08,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:09,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:11,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:11,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:11,094 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:12,383 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:15,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:20,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:23,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:23,056 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:23,056 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:23,056 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:23,387 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:23,608 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:20:24,387 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:25,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:26,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:26,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:26,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:28,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:31,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:36,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:41,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:41,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:41,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:42,147 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:43,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:43,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:43,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:43,486 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:44,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:44,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:47,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:48,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:52,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:53,609 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:20:56,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:56,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:56,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:58,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:03,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:03,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:03,982 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:03,983 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:03,983 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:04,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:06,404 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:08,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:08,404 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:11,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:11,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:11,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:13,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:18,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:23,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:23,610 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:21:25,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:25,057 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:25,057 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:25,058 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:25,411 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:26,411 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:26,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:26,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:26,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:29,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:30,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:34,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:39,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:41,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:44,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:46,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:46,317 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:46,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:46,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:46,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:48,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:50,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:50,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:53,610 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:21:55,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:56,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:56,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:56,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:00,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:05,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:05,697 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:05,697 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:05,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:06,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:06,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:08,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:10,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:11,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:11,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:11,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:11,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:16,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:21,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:23,612 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:22:24,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:24,065 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:24,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:24,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:24,434 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:26,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:26,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:26,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:26,795 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:27,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:28,436 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:32,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:37,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:41,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:41,785 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:41,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:42,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:43,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:43,633 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:43,634 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:43,634 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:44,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:44,443 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:48,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:48,444 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:53,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:53,612 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:22:56,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:56,785 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:56,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:58,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:03,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:03,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:03,923 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:03,923 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:03,924 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:04,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:06,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:08,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:09,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:11,921 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:14,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:19,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:23,613 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:23:25,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:26,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:26,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:26,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:27,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:27,138 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:27,138 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:27,138 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:27,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:28,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:31,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:32,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:36,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:41,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:41,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:41,326 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:41,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:41,326 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:41,465 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:41,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:41,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:41,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:42,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:46,467 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:46,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:51,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:53,614 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:23:56,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:56,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:56,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:56,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:58,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:58,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:58,600 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:58,600 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:59,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:00,473 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:02,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:02,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:04,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:07,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:11,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:11,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:11,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:12,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:17,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:22,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:22,547 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:22,547 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:22,548 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:23,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:23,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:23,615 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:24:24,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:26,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:26,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:26,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:28,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:33,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:38,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:39,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:39,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:39,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:39,242 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:39,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:40,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:41,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:41,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:41,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:44,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:44,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:49,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:53,616 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:24:53,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:53,646 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:53,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:53,647 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:54,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:54,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:54,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:56,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:56,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:56,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:58,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:59,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:04,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:09,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:11,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:11,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:11,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:11,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:11,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:11,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:11,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:11,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:12,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:15,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:16,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:20,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:23,617 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:25:25,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:26,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:26,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:26,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:31,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:35,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:35,845 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:35,845 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:35,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:36,509 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:36,509 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:36,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:40,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:41,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:41,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:47,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:52,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:52,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:52,592 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:52,592 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:52,593 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:53,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:53,617 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:25:54,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:56,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:56,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:56,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:58,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:03,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:08,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:11,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:11,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:12,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:12,712 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:12,712 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:12,713 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:13,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:13,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:14,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:16,525 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:18,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:23,618 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:26:24,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:26,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:26,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:26,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:29,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:33,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:33,681 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:33,681 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:33,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:34,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:34,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:35,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:38,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:40,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:41,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:41,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:49,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:49,459 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:49,459 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:49,460 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:49,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:50,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:51,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:53,619 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:26:54,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:56,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:56,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:56,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:01,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:03,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:03,760 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:03,760 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:03,761 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:04,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:04,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:06,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:08,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:11,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:11,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:12,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:17,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:22,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:23,620 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:27:26,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:26,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:27,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:28,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:28,429 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:28,429 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:28,429 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:28,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:30,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:32,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:32,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:37,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:41,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:41,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:42,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:45,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:45,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:45,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:45,009 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:45,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:46,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:48,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:48,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:53,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:53,621 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:27:56,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:56,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:58,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:02,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:02,177 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:02,178 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:02,178 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:04,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:06,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:09,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:11,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:11,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:15,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:20,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:20,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:20,938 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:20,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:20,939 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:21,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:22,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:23,622 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:28:24,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:25,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:26,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:26,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:26,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:30,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:35,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:37,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:37,921 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:37,922 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:37,922 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:38,585 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:38,585 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:40,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:41,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:41,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:42,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:45,997 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:50,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:53,624 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:28:56,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:56,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:56,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:58,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:58,802 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:58,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:58,803 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:59,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:01,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:01,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:03,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:07,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:11,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:11,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:11,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:12,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:17,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:21,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:29:21,224 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:29:21,224 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:29:21,225 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:29:21,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:23,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:23,605 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:23,625 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:29:25,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:26,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:26,913 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:27,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:28,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:33,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:38,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:41,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:41,913 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:44,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:44,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:29:44,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:29:44,315 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:29:44,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:29:44,613 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:45,613 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:49,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:49,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:53,626 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:29:54,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:56,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:56,943 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:56,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:00,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:02,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:02,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:02,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:02,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:02,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:03,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:05,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:07,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:10,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:11,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:11,929 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:11,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:16,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:21,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:22,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:22,981 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:22,981 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:22,982 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:23,627 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:30:23,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:25,631 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:26,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:26,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:26,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:26,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:27,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:32,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:37,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:38,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:38,665 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:38,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:38,666 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:39,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:41,638 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:41,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:41,932 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:41,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:42,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:43,639 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:47,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:53,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:53,628 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:30:54,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:54,214 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:54,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:54,214 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:54,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:55,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:57,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:57,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:57,017 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:58,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:59,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:03,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:08,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:11,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:11,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:11,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:13,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:13,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:13,023 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:13,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:31:13,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:31:14,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:15,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:17,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:19,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:23,629 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:31:24,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:26,952 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:26,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:29,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:34,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:39,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:42,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:42,368 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:42,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:42,369 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:31:42,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:31:43,669 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:45,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:47,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:50,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:53,630 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:31:55,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:56,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:59,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:59,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:59,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:59,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:00,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:00,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:01,676 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:05,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:05,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:10,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:11,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:16,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:20,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:32:20,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:32:20,872 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:32:20,872 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:21,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:21,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:23,631 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:32:23,683 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:25,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:26,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:26,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:32,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:37,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:41,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:41,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:41,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:43,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:43,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:32:43,772 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:32:43,773 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:32:43,773 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:44,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:45,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:48,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:49,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:53,632 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:32:54,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:56,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:00,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:04,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:04,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:04,130 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:04,130 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:04,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:05,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:05,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:09,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:10,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:11,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:11,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:11,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:16,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:20,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:20,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:20,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:20,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:20,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:21,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:21,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:23,633 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:33:25,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:26,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:26,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:26,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:31,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:36,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:36,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:36,395 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:36,395 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:36,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:37,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:37,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:41,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:41,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:43,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:48,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:53,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:53,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:53,603 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:53,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:53,604 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:53,633 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:33:53,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:55,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:56,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:56,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:57,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:59,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:59,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:04,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:09,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:10,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:10,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:10,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:10,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:11,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:12,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:13,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:14,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:15,729 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:19,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:23,634 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:34:24,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:27,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:27,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:27,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:27,087 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:27,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:27,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:27,602 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:27,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:29,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:29,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:31,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:35,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:40,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:42,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:42,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:45,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:47,024 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:47,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:47,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:47,026 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:47,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:49,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:51,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:51,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:53,635 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:34:56,636 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:57,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:57,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:57,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:01,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:06,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:07,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:07,221 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:07,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:07,221 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:07,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:09,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:11,751 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:12,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:12,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:12,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:17,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:22,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:23,636 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:35:27,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:27,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:27,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:27,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:32,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:33,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:33,064 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:33,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:33,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:33,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:35,759 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:37,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:38,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:42,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:42,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:43,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:48,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:50,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:50,216 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:50,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:50,216 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:50,765 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:51,766 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:53,637 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:35:54,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:55,767 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:57,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:57,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:57,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:59,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:04,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:09,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:12,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:12,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:12,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:12,701 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:12,701 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:12,702 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:12,774 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:13,774 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:15,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:17,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:20,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:23,638 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:36:26,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:27,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:27,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:27,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:27,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:27,888 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:27,888 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:27,888 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:28,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:29,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:31,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:32,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:33,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:37,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:42,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:42,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:42,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:42,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:44,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:44,658 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:44,658 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:44,658 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:44,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:45,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:47,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:49,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:53,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:53,639 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:36:57,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:57,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:57,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:58,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:03,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:04,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:04,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:04,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:04,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:04,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:05,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:09,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:09,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:12,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:12,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:12,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:14,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:19,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:20,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:20,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:20,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:20,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:20,800 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:21,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:23,639 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:37:25,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:25,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:27,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:27,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:27,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:30,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:35,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:40,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:40,565 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:40,565 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:40,566 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:40,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:41,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:41,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:43,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:43,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:43,226 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:45,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:46,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:51,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:53,640 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:37:57,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:58,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:58,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:58,264 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:58,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:58,721 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:58,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:58,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:58,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:59,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:02,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:03,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:07,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:12,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:13,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:13,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:13,229 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:18,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:21,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:21,386 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:21,386 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:21,386 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:21,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:23,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:23,641 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:38:23,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:25,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:28,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:28,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:28,229 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:29,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:34,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:39,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:40,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:40,895 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:40,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:40,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:41,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:41,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:43,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:43,558 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:44,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:47,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:50,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:53,642 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:38:55,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:58,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:58,217 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:58,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:58,217 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:58,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:58,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:58,556 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:58,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:59,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:01,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:03,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:06,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:11,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:13,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:13,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:13,558 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:16,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:21,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:22,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:39:22,985 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:39:22,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:39:22,986 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:39:23,643 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:39:23,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:39:23,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:27,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:27,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:28,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:28,556 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:28,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:32,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:37,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:41,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:39:41,028 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:39:41,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:39:41,029 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:39:41,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:39:41,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:43,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:43,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:43,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:43,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:45,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:48,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:53,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:53,647 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:39:58,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:58,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:58,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:58,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:00,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:00,306 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:00,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:00,306 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:00,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:01,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:03,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:05,867 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:08,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:13,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:13,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:13,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:13,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:18,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:23,644 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:40:24,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:28,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:28,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:28,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:29,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:31,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:31,726 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:31,726 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:31,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:31,879 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:33,880 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:34,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:35,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:39,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:43,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:43,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:43,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:45,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:47,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:47,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:47,484 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:47,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:47,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:49,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:50,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:51,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:53,645 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:40:56,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:58,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:58,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:58,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:01,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:03,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:03,917 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:03,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:03,920 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:04,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:05,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:06,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:07,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:12,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:13,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:13,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:13,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:17,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:22,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:23,646 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:41:27,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:27,707 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:27,708 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:27,708 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:27,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:28,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:29,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:29,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:29,903 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:31,904 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:33,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:38,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:43,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:44,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:44,446 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:44,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:46,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:46,827 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:46,828 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:46,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:46,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:47,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:49,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:51,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:53,647 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:41:55,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:59,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:01,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:06,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:06,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:06,909 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:06,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:06,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:06,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:07,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:11,920 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:12,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:14,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:14,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:14,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:17,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:22,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:22,333 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:22,333 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:22,336 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:22,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:23,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:23,648 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:42:23,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:25,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:28,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:29,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:29,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:33,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:38,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:40,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:40,502 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:40,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:40,502 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:40,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:41,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:43,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:44,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:44,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:44,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:45,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:49,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:53,649 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:42:54,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:59,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:59,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:59,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:00,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:02,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:02,762 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:02,763 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:02,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:02,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:03,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:05,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:07,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:11,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:14,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:14,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:14,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:16,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:21,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:22,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:22,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:22,153 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:22,153 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:22,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:23,649 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:43:23,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:25,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:27,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:29,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:29,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:32,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:36,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:36,730 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:36,730 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:36,730 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:36,955 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:37,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:37,955 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:41,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:42,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:44,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:44,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:44,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:48,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:53,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:53,650 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:43:56,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:56,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:56,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:56,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:56,963 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:57,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:58,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:59,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:59,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:59,526 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:01,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:03,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:08,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:11,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:11,402 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:11,402 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:11,402 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:11,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:11,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:14,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:14,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:14,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:14,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:15,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:19,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:23,652 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:44:24,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:26,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:26,238 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:26,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:26,239 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:26,975 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:27,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:29,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:29,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:29,458 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:29,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:30,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:35,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:40,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:44,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:44,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:44,477 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:46,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:49,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:49,139 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:49,139 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:49,139 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:49,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:49,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:52,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:53,652 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:44:53,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:57,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:59,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:59,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:59,464 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:03,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:07,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:07,085 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:07,085 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:07,086 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:07,991 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:07,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:09,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:11,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:14,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:14,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:14,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:14,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:19,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:23,654 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:45:24,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:26,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:26,862 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:26,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:26,863 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:26,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:27,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:29,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:29,585 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:29,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:29,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:32,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:34,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:39,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:44,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:44,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:45,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:45,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:45,975 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:45,976 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:45,976 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:46,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:48,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:50,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:51,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:53,655 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:45:56,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:59,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:59,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:59,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:02,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:06,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:06,426 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:06,426 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:06,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:07,014 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:08,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:08,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:10,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:13,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:14,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:14,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:14,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:19,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:23,656 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:46:24,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:26,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:26,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:26,654 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:26,654 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:27,021 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:28,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:29,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:29,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:29,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:29,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:30,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:34,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:39,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:44,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:44,034 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:44,034 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:44,035 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:44,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:44,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:45,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:45,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:46,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:48,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:51,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:53,656 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:46:56,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:59,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:59,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:59,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:02,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:04,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:04,463 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:04,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:04,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:05,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:07,039 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:07,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:09,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:12,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:14,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:14,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:14,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:18,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:23,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:23,657 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:47:26,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:26,770 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:26,770 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:26,771 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:27,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:28,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:29,048 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:29,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:29,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:29,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:31,049 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:33,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:38,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:42,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:42,476 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:42,476 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:42,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:43,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:45,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:45,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:45,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:45,102 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:47,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:49,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:53,658 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:47:55,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:58,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:58,476 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:58,477 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:58,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:59,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:00,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:00,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:00,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:01,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:01,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:03,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:06,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:11,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:14,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:14,801 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:14,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:14,802 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:15,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:15,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:15,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:15,178 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:17,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:17,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:21,079 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:23,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:23,659 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:48:28,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:30,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:30,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:30,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:34,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:34,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:34,632 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:34,632 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:34,632 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:35,084 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:37,085 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:39,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:39,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:44,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:45,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:45,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:45,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:49,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:49,295 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:49,295 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:49,295 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:50,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:50,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:51,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:53,660 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:48:55,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:55,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:00,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:00,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:00,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:01,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:06,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:11,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:14,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:14,004 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:14,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:14,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:14,100 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:15,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:15,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:15,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:15,755 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:16,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:19,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:22,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:23,661 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:49:27,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:30,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:30,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:30,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:31,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:31,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:31,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:31,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:32,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:32,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:33,108 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:37,110 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:37,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:42,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:45,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:45,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:45,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:48,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:49,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:49,310 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:49,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:49,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:50,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:51,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:53,662 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:49:54,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:55,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:59,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:00,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:00,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:00,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:06,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:06,805 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:06,805 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:06,805 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:07,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:09,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:09,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:11,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:14,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:16,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:16,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:16,064 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:20,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:22,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:22,630 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:22,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:22,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:23,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:23,663 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:50:25,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:25,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:27,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:30,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:31,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:31,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:36,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:39,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:39,764 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:39,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:39,764 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:40,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:41,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:41,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:45,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:46,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:46,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:46,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:47,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:52,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:53,664 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:50:54,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:54,082 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:54,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:54,083 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:54,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:55,139 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:57,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:59,140 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:01,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:01,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:01,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:03,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:08,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:10,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:10,199 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:10,199 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:10,199 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:11,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:11,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:13,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:15,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:16,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:16,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:16,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:19,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:23,665 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:51:24,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:29,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:30,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:30,413 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:30,413 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:30,413 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:31,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:31,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:31,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:33,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:35,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:35,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:40,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:45,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:46,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:46,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:46,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:49,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:49,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:49,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:49,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:50,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:51,160 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:51,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:53,665 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:51:55,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:56,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:01,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:01,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:01,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:02,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:03,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:03,594 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:03,595 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:03,595 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:04,164 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:05,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:07,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:09,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:12,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:16,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:16,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:16,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:18,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:21,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:21,172 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:21,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:21,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:22,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:23,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:23,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:23,666 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:52:25,172 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:28,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:31,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:31,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:34,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:39,112 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:40,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:40,661 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:40,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:40,662 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:41,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:43,178 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:44,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:45,179 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:46,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:46,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:46,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:50,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:53,667 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:52:55,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:00,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:01,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:01,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:01,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:03,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:03,244 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:03,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:03,245 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:04,186 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:05,187 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:06,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:07,188 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:11,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:16,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:16,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:16,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:17,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:22,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:23,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:23,022 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:23,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:23,023 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:23,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:23,668 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:53:25,194 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:27,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:27,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:31,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:31,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:31,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:33,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:38,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:40,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:40,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:40,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:40,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:41,201 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:43,202 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:43,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:45,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:46,011 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:46,011 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:46,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:49,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:53,669 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:53:54,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:58,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:58,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:58,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:58,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:59,216 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:59,216 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:00,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:01,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:01,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:01,074 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:03,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:05,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:10,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:12,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:12,993 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:12,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:12,994 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:13,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:15,222 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:16,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:16,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:16,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:16,033 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:17,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:21,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:23,670 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:54:26,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:30,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:30,974 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:30,975 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:30,975 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:31,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:31,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:31,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:31,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:32,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:33,229 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:35,230 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:37,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:37,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:42,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:46,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:46,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:46,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:47,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:47,556 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:47,557 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:47,557 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:48,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:48,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:49,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:53,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:53,671 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:54:53,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:58,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:01,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:01,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:01,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:04,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:09,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:09,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:09,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:09,337 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:09,338 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:10,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:11,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:13,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:14,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:16,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:16,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:16,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:20,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:23,672 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:55:25,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:26,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:26,720 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:26,720 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:26,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:27,251 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:29,252 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:30,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:31,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:31,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:31,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:31,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:36,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:41,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:45,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:45,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:45,798 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:45,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:46,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:46,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:46,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:46,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:47,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:47,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:51,261 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:53,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:53,673 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:55:58,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:01,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:01,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:01,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:01,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:01,667 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:01,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:01,668 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:02,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:03,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:03,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:05,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:07,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:08,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:13,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:16,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:16,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:16,042 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:19,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:23,674 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:56:23,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:23,832 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:23,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:23,833 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:24,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:24,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:25,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:29,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:29,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:31,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:31,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:31,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:35,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:40,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:40,122 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:40,122 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:40,122 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:40,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:40,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:41,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:45,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:45,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:46,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:46,037 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:51,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:53,675 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:56:56,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:57,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:57,840 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:57,841 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:57,841 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:58,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:59,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:01,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:01,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:01,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:02,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:03,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:07,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:12,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:16,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:16,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:16,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:18,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:19,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:19,698 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:19,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:19,699 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:20,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:21,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:23,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:23,676 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:57:23,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:25,298 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:28,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:31,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:31,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:31,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:34,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:35,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:35,578 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:35,578 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:35,579 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:36,302 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:37,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:39,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:39,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:44,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:46,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:46,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:50,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:53,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:53,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:53,024 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:53,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:53,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:53,676 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:57:55,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:55,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:57,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:01,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:01,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:01,075 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:01,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:06,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:10,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:10,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:10,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:10,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:10,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:11,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:11,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:15,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:16,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:16,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:16,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:17,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:22,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:23,677 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:58:27,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:31,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:31,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:31,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:31,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:31,123 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:31,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:31,124 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:31,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:33,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:33,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:35,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:38,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:46,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:46,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:46,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:48,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:48,621 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:48,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:48,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:48,622 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:49,329 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:49,329 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:53,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:53,679 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:58:53,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:58,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:01,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:01,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:01,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:04,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:09,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:09,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:09,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:09,392 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:09,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:10,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:11,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:13,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:14,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:16,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:16,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:16,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:20,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:23,680 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:59:25,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:28,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:28,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:28,054 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:28,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:29,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:31,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:31,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:31,064 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:31,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:33,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:36,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:41,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:46,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:46,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:46,061 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:47,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:47,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:47,054 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:47,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:47,055 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:47,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:49,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:51,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:52,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:53,680 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:59:57,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:01,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:01,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:01,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:03,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:04,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:04,564 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:04,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:04,565 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:05,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:05,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:08,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:09,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:13,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:16,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:16,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:16,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:19,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:21,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:21,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:21,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:21,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:21,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:21,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:23,681 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:00:24,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:25,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:29,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:31,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:31,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:31,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:35,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:40,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:43,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:43,526 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:43,526 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:43,527 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:44,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:45,374 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:45,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:46,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:46,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:46,195 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:47,375 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:51,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:53,683 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:00:56,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:01,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:01,196 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:01,199 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:02,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:03,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:03,759 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:03,760 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:03,760 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:04,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:05,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:07,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:07,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:12,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:16,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:16,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:16,193 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:18,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:23,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:23,684 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:01:28,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:29,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:29,263 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:29,263 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:29,263 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:29,390 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:31,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:31,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:31,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:33,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:34,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:39,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:44,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:46,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:49,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:53,685 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:01:54,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:54,631 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:54,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:54,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:55,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:55,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:55,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:59,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:01,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:01,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:06,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:11,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:16,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:16,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:16,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:17,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:19,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:02:19,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:02:19,023 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:02:19,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:02:19,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:02:19,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:23,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:23,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:23,686 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:02:28,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:31,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:31,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:31,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:33,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:38,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:41,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:02:41,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:02:41,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:02:41,444 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:02:42,418 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:02:43,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:44,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:45,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:49,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:53,687 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:02:55,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:00,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:01,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:01,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:01,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:04,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:04,535 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:04,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:04,535 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:05,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:05,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:06,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:09,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:11,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:16,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:16,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:16,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:18,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:18,467 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:18,468 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:18,468 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:19,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:19,434 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:22,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:23,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:23,688 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:03:27,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:31,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:31,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:31,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:32,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:36,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:36,052 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:36,052 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:36,053 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:36,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:37,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:38,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:39,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:43,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:46,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:46,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:48,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:53,694 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:03:54,695 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:56,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:56,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:56,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:56,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:57,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:57,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:59,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:01,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:01,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:01,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:04,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:09,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:13,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:04:13,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:04:13,445 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:04:13,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:04:13,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:04:15,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:15,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:16,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:16,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:16,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:17,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:20,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:23,690 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:04:26,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:31,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:31,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:36,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:37,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:04:37,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:04:37,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:04:37,446 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:04:37,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:04:39,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:41,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:42,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:46,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:47,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:52,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:53,690 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:04:58,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:01,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:01,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:02,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:02,623 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:02,623 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:02,623 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:03,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:04,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:04,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:08,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:09,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:16,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:16,866 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:16,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:19,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:23,691 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:05:24,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:24,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:24,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:24,345 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:24,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:25,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:26,478 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:30,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:30,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:35,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:40,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:41,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:41,091 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:41,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:41,093 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:41,483 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:44,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:46,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:46,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:46,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:51,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:53,693 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:05:57,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:01,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:02,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:03,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:03,645 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:03,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:03,646 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:04,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:06,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:08,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:08,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:13,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:16,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:16,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:18,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:21,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:21,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:21,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:21,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:21,498 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:23,693 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:06:24,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:24,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:26,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:29,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:34,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:39,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:39,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:39,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:39,554 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:40,505 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:40,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:42,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:44,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:45,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:46,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:51,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:06:56,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:01,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:01,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:03,105 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:03,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:03,106 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:03,108 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:03,109 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:03,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:04,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:06,515 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:07,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:08,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:12,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:13,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:18,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:19,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:19,209 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:19,210 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:19,210 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:19,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:22,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:22,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:23,696 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:23,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:24,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:27,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:28,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:31,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:33,698 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:37,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:38,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:41,954 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:41,956 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:42,527 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:42,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:43,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:44,528 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:46,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:46,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:49,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:52,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:53,697 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:54,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:57,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:58,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:58,792 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:58,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:58,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:59,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:00,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:01,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:04,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:04,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:07,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:09,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:12,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:14,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:16,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:17,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:17,307 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:17,307 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:17,307 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:17,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:08:20,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:20,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:22,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:22,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:23,697 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:08:25,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:27,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:30,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:35,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:36,962 WARNING FileStreamThread:1091 [file_stream.py:request_with_retry():687] requests_with_retry encountered retryable exception: 500 Server Error: Internal Server Error for url: https://api.wandb.ai/files/ducminh-ml71/huggingface/rimyh9a1/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/ducminh-ml71/huggingface/rimyh9a1/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"train/loss": 3.3958, "train/grad_norm": 99.48656463623047, "train/learning_rate": 1.2869649049440873e-05, "train/epoch": 0.05, "train/global_step": 11180, "_timestamp": 1712819261.9524338, "_runtime": 10819.103787899017, "_step": 564}']}, 'wandb-history.jsonl': {'offset': 562, 'content': ['{"train/loss": 3.2126, "train/grad_norm": 58.60544967651367, "train/learning_rate": 1.2965870820261672e-05, "train/epoch": 0.05, "train/global_step": 11140, "_timestamp": 1712819221.2956657, "_runtime": 10778.447019815445, "_step": 562}', '{"train/loss": 4.4423, "train/grad_norm": 15.332870483398438, "train/learning_rate": 1.2917749013937503e-05, "train/epoch": 0.05, "train/global_step": 11160, "_timestamp": 1712819239.2079432, "_runtime": 10796.359297275543, "_step": 563}', '{"train/loss": 3.3958, "train/grad_norm": 99.48656463623047, "train/learning_rate": 1.2869649049440873e-05, "train/epoch": 0.05, "train/global_step": 11180, "_timestamp": 1712819261.9524338, "_runtime": 10819.103787899017, "_step": 564}']}, 'wandb-events.jsonl': {'offset': 358, 'content': ['{"system.network.sent": 676280115.93, "system.network.recv": 452830156.0, "system.proc.memory.availableMB": 28276.28, "system.memory": 11.95, "system.proc.memory.rssMB": 2828.42, "system.proc.memory.percent": 8.81, "system.cpu": 28.48, "system.cpu.0.cpu_percent": 25.15, "system.cpu.1.cpu_percent": 61.31, "system.cpu.2.cpu_percent": 15.83, "system.cpu.3.cpu_percent": 21.72, "system.proc.cpu.threads": 43, "system.disk.\\\\.usagePercent": 69.0, "system.disk.\\\\.usageGB": 5566.96, "system.disk.in": 229.0, "system.disk.out": 976402.3, "system.gpu.0.memoryAllocated": 84.13, "system.gpu.1.memoryAllocated": 28.55, "system.gpu.0.memoryAllocatedBytes": 13550682112.0, "system.gpu.1.memoryAllocatedBytes": 4597940224.0, "system.gpu.0.memory": 33.13, "system.gpu.1.memory": 1.93, "system.gpu.0.gpu": 59.8, "system.gpu.1.gpu": 21.33, "system.gpu.0.temp": 76.93, "system.gpu.1.temp": 55.6, "system.gpu.0.powerWatts": 62.3, "system.gpu.1.powerWatts": 33.69, "system.gpu.0.powerPercent": 89.55, "system.gpu.1.powerPercent": 47.98, "_wandb": true, "_timestamp": 1712819243.694884, "_runtime": 10800.846238}']}, 'output.log': {'offset': 3380, 'content': ['ERROR 2024-04-11T07:07:03.631961 /opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n', 'ERROR 2024-04-11T07:07:03.631961 warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.")\n', 'ERROR 2024-04-11T07:07:05.636647 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n', 'ERROR 2024-04-11T07:07:05.636647 warnings.warn(\n', 'ERROR 2024-04-11T07:07:05.636647 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n', "ERROR 2024-04-11T07:07:05.636647 warnings.warn('Was asked to gather along dimension 0, but all '\n", 'ERROR 2024-04-11T07:07:21.656253 /opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n', 'ERROR 2024-04-11T07:07:21.656253 warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.")\n', 'ERROR 2024-04-11T07:07:23.661960 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n', 'ERROR 2024-04-11T07:07:23.661960 warnings.warn(\n', 'ERROR 2024-04-11T07:07:23.661960 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n', "ERROR 2024-04-11T07:07:23.661960 warnings.warn('Was asked to gather along dimension 0, but all '\n"]}}, 'dropped': 0}} +2024-04-11 07:08:37,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:38,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:38,651 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:38,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:38,651 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:39,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:08:40,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:41,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:42,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:44,551 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:46,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:46,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:51,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:52,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:53,698 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:08:57,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:57,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:57,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:57,896 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:57,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:57,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:58,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:00,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:01,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:02,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:03,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:07,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:08,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:12,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:13,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:16,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:18,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:18,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:09:18,683 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:09:18,683 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:09:18,684 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:09:19,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:20,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:22,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:23,699 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:09:23,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:24,566 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:27,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:28,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:33,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:37,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:38,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:40,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:09:40,762 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:09:40,763 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:09:40,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:09:41,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:42,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:42,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:43,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:46,574 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:49,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:52,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:53,701 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:09:54,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:57,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:59,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:01,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:03,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:03,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:03,511 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:03,512 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:03,581 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:05,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:06,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:07,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:08,583 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:10,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:12,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:15,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:16,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:20,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:22,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:23,702 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:10:26,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:26,322 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:26,323 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:26,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:26,324 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:26,590 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:28,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:28,591 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:30,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:31,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:32,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:36,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:37,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:41,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:42,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:46,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:46,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:47,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:47,470 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:47,470 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:47,472 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:47,598 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:50,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:51,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:52,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:52,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:53,703 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:10:56,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:57,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:01,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:01,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:06,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:07,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:07,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:07,397 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:07,398 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:07,398 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:07,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:08,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:11,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:12,610 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:12,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:16,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:21,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:21,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:23,703 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:11:25,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:25,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:25,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:25,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:25,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:26,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:27,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:27,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:30,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:31,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:32,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:37,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:42,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:42,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:44,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:44,729 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:44,729 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:44,729 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:45,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:46,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:47,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:47,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:47,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:50,624 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:52,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:52,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:53,704 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:11:57,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:58,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:59,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:59,612 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:59,612 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:59,613 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:59,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:02,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:02,629 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:03,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:04,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:08,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:08,860 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:13,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:13,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:17,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:17,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:17,560 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:17,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:17,561 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:17,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:18,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:19,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:22,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:23,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:23,705 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:12:24,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:28,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:29,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:32,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:34,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:38,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:38,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:38,334 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:38,334 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:38,335 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:38,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:40,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:40,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:42,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:44,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:45,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:47,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:50,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:53,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:53,706 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:12:55,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:56,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:56,627 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:56,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:56,628 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:56,649 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:58,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:58,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:00,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:00,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:02,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:02,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:05,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:08,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:10,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:11,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:11,508 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:11,509 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:11,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:11,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:12,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:14,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:16,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:16,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:17,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:21,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:23,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:23,707 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:13:26,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:27,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:27,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:27,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:27,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:28,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:28,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:30,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:32,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:32,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:32,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:37,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:37,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:42,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:42,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:45,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:45,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:45,436 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:45,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:45,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:46,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:47,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:48,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:48,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:50,672 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:52,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:53,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:53,708 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:13:57,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:58,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:02,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:03,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:05,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:05,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:05,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:05,845 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:06,678 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:07,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:08,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:08,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:10,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:13,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:13,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:18,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:18,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:23,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:23,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:23,709 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:14:25,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:25,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:25,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:25,451 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:25,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:26,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:28,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:29,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:30,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:33,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:33,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:38,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:38,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:43,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:43,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:43,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:43,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:43,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:43,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:44,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:44,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:48,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:48,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:49,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:53,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:53,710 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:14:55,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:58,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:00,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:01,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:01,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:01,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:01,291 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:01,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:02,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:03,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:03,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:05,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:06,703 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:09,011 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:10,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:14,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:15,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:17,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:17,727 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:17,727 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:17,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:18,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:18,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:19,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:19,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:19,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:20,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:22,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:23,711 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:15:24,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:26,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:29,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:31,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:34,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:36,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:37,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:37,658 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:37,658 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:37,658 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:37,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:38,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:40,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:41,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:42,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:45,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:46,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:49,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:51,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:53,712 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:15:55,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:55,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:55,639 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:55,639 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:55,639 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:55,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:56,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:57,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:00,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:00,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:02,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:04,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:07,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:10,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:12,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:15,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:16,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:16,166 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:16,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:16,167 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:16,729 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:18,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:18,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:19,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:20,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:23,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:23,713 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:16:24,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:28,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:29,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:30,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:30,016 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:30,016 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:30,016 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:30,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:32,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:34,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:34,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:34,736 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:39,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:39,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:44,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:44,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:47,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:47,263 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:47,263 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:47,263 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:47,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:48,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:49,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:49,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:49,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:52,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:53,713 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:16:54,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:54,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:59,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:00,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:03,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:03,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:03,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:03,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:04,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:04,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:04,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:05,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:05,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:05,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:08,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:10,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:10,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:15,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:15,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:20,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:20,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:23,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:23,566 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:23,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:23,567 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:23,714 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:17:23,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:24,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:25,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:26,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:28,757 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:30,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:31,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:35,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:35,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:38,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:38,039 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:38,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:38,040 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:38,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:38,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:40,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:41,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:42,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:45,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:46,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:50,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:51,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:53,715 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:17:55,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:56,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:57,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:57,147 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:57,148 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:57,148 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:57,768 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:58,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:00,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:02,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:02,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:05,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:07,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:10,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:12,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:15,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:17,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:19,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:19,614 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:19,614 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:19,614 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:19,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:20,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:21,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:22,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:23,716 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:18:24,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:26,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:27,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:31,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:32,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:35,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:35,948 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:35,948 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:35,948 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:36,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:36,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:36,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:37,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:40,785 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:41,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:43,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:46,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:48,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:50,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:50,558 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:50,559 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:50,559 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:50,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:51,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:52,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:53,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:53,722 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:18:54,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:57,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:58,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:02,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:03,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:06,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:08,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:10,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:10,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:10,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:10,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:11,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:12,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:12,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:13,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:14,798 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:17,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:19,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:23,718 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:19:24,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:26,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:27,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:27,495 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:27,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:27,497 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:27,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:28,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:30,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:32,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:32,805 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:35,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:36,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:40,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:41,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:41,102 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:41,103 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:41,103 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:41,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:42,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:42,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:44,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:46,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:47,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:51,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:51,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:53,719 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:19:56,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:56,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:01,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:01,519 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:01,519 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:01,521 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:01,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:01,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:02,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:02,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:06,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:06,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:07,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:12,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:12,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:17,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:17,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:21,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:22,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:22,109 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:22,109 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:22,110 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:22,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:22,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:23,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:23,720 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:20:26,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:26,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:28,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:31,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:33,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:36,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:38,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:39,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:39,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:39,133 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:39,135 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:39,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:40,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:41,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:42,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:43,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:46,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:48,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:51,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:53,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:53,721 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:20:56,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:58,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:00,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:00,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:00,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:00,112 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:00,843 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:00,844 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:01,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:04,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:04,845 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:06,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:09,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:11,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:14,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:16,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:16,996 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:16,996 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:16,998 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:17,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:18,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:20,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:20,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:21,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:23,722 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:21:25,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:27,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:30,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:32,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:35,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:36,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:38,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:38,434 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:38,434 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:38,435 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:38,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:40,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:41,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:42,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:42,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:46,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:47,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:51,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:51,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:53,723 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:21:55,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:55,461 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:55,461 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:55,462 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:55,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:56,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:57,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:57,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:00,868 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:02,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:02,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:06,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:07,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:12,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:12,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:14,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:14,486 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:14,486 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:14,487 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:14,873 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:16,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:17,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:18,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:18,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:21,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:23,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:23,724 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:22:27,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:28,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:32,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:32,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:32,942 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:32,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:32,943 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:33,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:33,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:34,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:36,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:36,886 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:39,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:41,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:44,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:46,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:49,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:51,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:53,725 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:22:54,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:56,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:57,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:57,092 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:57,092 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:57,092 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:57,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:59,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:00,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:01,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:01,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:05,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:06,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:10,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:10,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:10,742 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:10,742 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:10,743 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:10,899 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:12,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:13,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:15,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:15,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:17,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:20,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:21,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:23,725 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:23:26,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:26,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:31,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:31,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:32,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:32,520 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:32,520 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:32,520 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:32,907 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:33,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:36,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:37,698 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:37,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:42,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:42,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:47,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:47,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:47,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:47,799 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:47,799 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:47,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:47,913 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:49,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:51,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:52,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:53,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:55,332 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:23:56,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:58,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:01,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:02,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:02,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:02,777 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:02,777 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:02,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:03,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:05,920 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:06,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:07,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:08,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:11,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:13,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:16,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:18,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:21,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:22,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:22,833 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:22,833 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:22,834 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:22,927 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:23,727 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:24:24,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:25,928 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:26,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:27,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:30,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:31,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:35,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:36,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:40,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:41,567 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:42,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:42,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:42,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:42,653 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:42,934 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:43,935 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:45,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:47,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:47,936 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:50,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:51,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:53,728 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:24:56,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:56,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:01,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:01,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:02,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:02,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:02,600 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:02,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:02,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:03,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:06,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:06,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:07,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:11,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:12,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:16,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:17,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:21,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:21,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:22,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:22,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:22,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:22,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:22,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:23,729 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:25:23,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:26,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:27,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:27,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:31,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:32,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:36,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:37,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:41,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:41,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:41,907 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:41,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:41,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:41,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:42,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:43,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:47,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:47,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:48,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:51,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:53,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:53,730 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:25:56,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:58,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:01,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:03,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:03,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:03,394 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:03,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:03,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:04,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:05,965 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:06,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:07,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:09,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:12,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:14,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:17,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:19,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:21,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:23,731 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:26:24,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:24,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:24,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:24,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:24,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:25,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:25,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:26,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:29,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:30,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:31,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:35,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:36,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:40,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:41,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:44,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:44,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:44,338 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:44,338 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:44,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:45,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:46,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:47,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:49,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:51,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:51,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:53,732 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:26:56,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:57,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:01,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:02,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:02,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:02,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:02,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:02,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:03,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:06,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:07,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:07,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:11,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:12,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:16,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:17,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:21,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:21,024 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:21,024 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:21,025 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:21,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:21,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:23,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:23,733 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:27:23,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:25,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:26,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:28,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:31,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:33,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:36,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:38,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:41,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:42,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:42,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:42,529 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:42,529 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:43,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:44,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:47,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:48,002 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:48,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:51,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:53,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:53,733 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:27:57,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:58,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:02,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:03,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:05,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:05,932 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:05,933 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:05,933 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:06,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:06,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:08,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:08,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:11,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:12,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:14,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:16,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:19,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:21,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:23,735 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:28:24,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:26,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:26,598 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:26,598 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:26,599 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:26,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:27,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:28,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:30,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:32,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:32,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:35,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:36,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:40,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:42,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:45,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:45,116 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:45,116 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:45,116 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:46,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:46,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:47,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:48,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:50,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:51,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:51,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:53,736 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:28:56,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:56,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:01,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:01,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:04,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:04,879 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:04,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:04,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:05,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:06,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:06,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:06,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:10,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:11,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:12,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:16,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:17,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:21,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:22,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:23,737 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:29:24,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:24,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:24,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:24,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:25,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:26,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:27,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:27,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:30,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:32,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:32,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:36,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:37,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:42,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:42,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:43,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:43,557 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:43,557 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:43,558 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:44,045 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:46,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:47,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:47,703 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:48,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:51,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:52,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:53,737 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:29:57,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:57,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:02,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:02,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:07,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:09,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:09,779 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:09,779 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:09,779 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:10,055 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:12,056 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:12,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:12,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:14,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:17,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:17,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:21,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:22,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:23,738 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:30:27,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:28,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:30,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:30,284 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:30,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:30,285 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:31,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:32,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:33,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:34,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:36,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:39,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:42,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:44,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:47,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:49,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:51,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:53,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:53,192 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:53,193 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:53,193 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:53,739 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:30:54,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:54,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:56,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:56,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:58,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:00,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:01,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:05,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:06,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:10,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:10,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:10,517 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:10,517 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:10,517 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:11,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:12,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:12,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:15,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:16,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:17,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:20,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:23,740 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:31:26,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:27,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:30,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:30,827 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:30,827 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:30,828 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:31,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:31,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:32,089 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:32,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:36,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:36,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:42,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:46,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:47,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:51,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:51,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:52,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:52,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:52,337 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:52,337 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:53,096 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:53,741 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:31:54,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:57,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:57,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:58,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:02,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:02,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:07,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:12,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:12,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:15,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:15,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:15,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:15,314 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:16,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:16,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:17,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:18,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:20,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:23,742 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:32:27,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:28,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:29,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:29,474 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:29,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:31,337 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:32,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:32,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:32,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:34,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:36,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:39,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:42,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:44,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:47,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:48,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:48,719 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:48,719 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:48,720 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:49,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:49,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:50,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:51,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:53,742 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:32:54,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:55,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:57,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:00,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:02,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:05,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:08,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:08,975 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:08,975 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:08,975 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:09,124 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:10,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:11,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:11,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:14,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:16,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:16,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:21,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:21,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:23,744 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:33:26,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:26,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:29,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:29,245 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:29,246 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:29,246 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:30,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:30,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:31,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:32,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:34,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:36,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:37,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:41,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:42,419 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:46,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:47,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:51,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:51,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:51,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:51,345 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:51,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:52,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:52,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:53,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:53,744 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:33:56,144 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:57,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:58,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:02,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:03,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:06,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:07,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:07,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:07,979 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:07,979 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:08,148 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:09,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:10,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:11,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:12,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:14,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:16,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:19,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:21,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:22,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:22,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:22,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:22,283 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:23,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:24,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:24,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:25,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:34:27,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:28,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:29,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:32,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:34,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:36,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:39,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:39,769 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:39,769 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:39,770 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:40,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:40,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:42,162 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:42,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:44,163 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:45,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:47,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:50,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:51,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:53,746 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:34:56,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:56,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:56,838 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:56,838 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:56,840 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:57,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:57,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:58,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:02,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:02,170 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:02,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:06,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:07,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:12,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:12,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:15,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:15,389 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:15,389 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:15,390 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:16,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:16,176 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:17,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:18,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:20,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:21,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:22,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:23,747 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:35:27,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:28,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:32,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:33,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:34,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:34,382 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:34,382 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:34,384 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:35,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:36,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:36,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:38,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:39,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:42,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:44,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:47,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:49,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:51,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:52,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:52,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:52,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:52,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:52,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:53,748 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:35:54,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:54,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:56,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:57,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:00,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:02,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:05,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:06,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:06,817 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:06,818 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:06,820 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:06,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:07,195 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:08,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:11,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:12,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:12,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:16,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:17,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:21,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:21,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:23,749 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:36:26,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:26,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:26,365 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:26,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:26,367 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:27,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:28,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:28,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:30,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:31,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:33,153 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:36,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:36,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:41,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:42,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:44,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:44,761 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:44,761 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:44,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:45,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:46,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:46,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:47,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:50,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:51,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:51,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:53,751 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:36:57,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:57,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:58,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:58,635 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:58,635 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:58,635 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:59,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:00,218 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:03,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:03,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:04,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:06,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:08,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:12,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:13,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:15,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:15,499 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:15,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:15,501 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:16,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:16,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:18,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:18,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:20,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:21,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:23,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:23,751 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:37:27,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:28,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:32,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:33,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:34,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:34,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:34,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:34,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:35,230 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:36,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:37,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:37,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:38,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:40,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:42,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:43,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:47,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:48,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:51,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:51,795 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:51,796 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:51,798 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:52,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:52,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:52,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:53,753 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:37:54,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:56,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:57,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:00,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:02,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:05,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:07,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:10,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:11,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:11,352 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:11,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:11,353 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:12,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:12,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:13,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:15,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:16,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:18,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:20,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:22,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:23,754 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:38:25,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:26,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:26,099 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:26,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:26,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:26,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:27,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:28,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:30,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:31,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:32,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:36,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:37,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:41,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:42,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:43,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:43,954 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:43,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:43,954 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:44,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:46,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:46,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:48,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:48,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:52,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:52,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:53,755 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:38:57,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:57,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:01,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:01,341 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:01,341 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:01,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:02,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:02,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:03,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:03,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:06,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:07,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:08,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:13,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:13,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:17,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:17,146 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:17,146 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:17,147 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:17,269 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:18,269 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:18,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:19,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:22,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:22,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:23,756 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:39:24,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:27,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:29,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:32,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:33,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:33,870 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:33,870 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:33,871 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:34,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:34,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:34,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:37,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:40,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:40,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:45,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:47,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:50,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:52,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:52,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:52,750 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:52,750 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:52,751 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:53,283 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:53,757 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:39:54,283 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:55,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:56,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:58,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:00,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:03,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:05,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:07,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:09,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:09,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:09,130 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:09,130 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:09,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:10,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:11,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:12,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:15,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:16,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:17,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:21,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:22,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:23,758 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:40:25,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:25,180 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:25,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:25,181 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:25,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:27,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:27,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:27,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:31,299 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:32,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:32,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:37,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:37,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:40,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:40,878 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:40,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:40,878 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:41,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:42,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:42,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:43,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:45,304 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:47,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:47,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:48,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:52,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:53,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:53,759 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:40:57,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:57,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:57,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:57,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:57,680 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:58,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:58,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:59,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:03,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:03,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:04,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:07,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:09,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:12,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:14,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:17,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:17,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:17,512 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:17,512 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:17,512 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:18,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:19,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:19,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:22,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:23,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:23,760 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:41:24,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:28,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:29,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:33,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:34,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:35,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:35,976 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:35,976 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:35,977 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:36,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:37,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:37,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:40,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:41,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:42,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:45,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:47,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:50,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:52,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:53,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:53,496 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:53,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:53,497 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:53,762 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:41:54,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:55,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:55,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:58,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:59,333 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:01,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:03,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:06,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:07,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:11,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:13,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:14,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:14,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:14,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:14,904 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:15,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:16,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:17,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:18,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:19,340 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:22,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:22,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:23,762 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:42:27,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:28,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:32,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:32,632 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:32,633 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:32,633 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:33,346 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:33,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:34,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:35,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:37,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:37,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:38,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:43,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:43,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:48,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:48,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:52,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:53,763 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:42:54,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:54,187 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:54,187 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:54,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:54,188 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:54,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:55,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:57,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:59,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:59,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:02,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:04,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:07,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:09,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:12,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:14,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:14,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:14,387 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:14,388 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:14,388 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:15,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:17,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:18,103 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:19,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:19,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:22,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:23,764 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:43:24,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:28,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:29,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:33,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:34,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:36,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:36,311 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:36,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:36,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:36,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:37,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:38,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:40,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:41,372 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:43,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:45,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:49,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:50,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:53,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:53,765 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:43:55,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:57,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:57,667 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:57,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:57,667 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:58,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:58,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:59,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:00,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:03,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:03,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:05,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:08,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:10,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:13,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:15,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:18,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:20,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:23,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:23,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:23,312 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:23,312 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:23,312 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:23,388 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:23,766 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:44:25,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:26,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:27,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:28,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:29,390 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:31,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:33,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:36,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:38,019 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:40,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:40,079 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:40,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:40,080 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:40,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:41,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:42,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:43,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:45,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:47,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:48,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:52,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:53,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:53,767 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:44:54,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:54,416 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:54,416 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:54,416 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:55,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:55,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:57,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:59,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:59,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:02,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:04,153 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:07,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:08,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:12,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:14,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:16,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:16,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:16,966 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:16,967 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:17,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:17,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:19,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:19,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:21,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:23,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:23,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:23,767 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:45:28,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:28,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:33,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:33,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:38,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:38,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:40,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:40,447 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:40,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:40,448 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:41,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:41,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:44,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:44,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:45,423 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:49,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:49,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:53,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:53,769 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:45:54,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:58,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:58,369 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:58,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:58,370 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:58,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:59,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:59,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:00,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:03,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:05,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:05,567 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:08,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:10,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:14,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:15,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:19,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:20,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:21,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:21,783 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:21,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:21,784 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:22,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:23,438 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:23,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:23,770 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:46:26,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:27,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:28,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:31,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:33,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:36,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:38,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:39,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:39,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:39,966 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:39,966 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:40,445 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:41,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:42,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:43,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:45,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:47,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:48,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:52,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:53,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:53,770 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:46:57,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:57,487 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:57,487 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:57,487 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:57,488 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:58,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:59,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:59,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:01,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:02,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:03,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:04,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:07,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:08,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:12,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:14,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:17,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:19,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:19,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:47:19,330 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:47:19,330 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:47:19,330 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:47:19,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:47:21,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:23,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:23,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:23,772 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:47:28,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:29,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:33,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:34,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:38,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:38,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:40,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:47:40,787 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:47:40,788 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:47:40,788 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:47:41,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:47:43,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:43,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:43,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:45,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:48,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:49,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:53,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:53,773 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:47:54,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:58,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:59,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:03,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:04,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:05,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:05,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:05,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:05,669 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:06,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:07,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:08,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:09,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:09,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:11,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:14,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:14,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:19,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:19,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:23,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:23,773 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:48:24,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:24,985 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:24,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:24,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:24,986 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:25,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:27,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:28,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:29,489 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:30,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:33,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:35,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:38,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:39,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:39,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:39,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:39,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:39,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:40,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:41,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:43,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:44,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:45,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:49,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:50,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:53,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:53,774 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:48:54,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:54,252 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:54,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:54,253 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:54,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:55,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:56,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:59,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:59,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:01,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:04,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:06,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:08,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:11,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:14,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:16,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:19,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:19,065 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:19,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:19,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:19,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:19,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:21,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:22,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:23,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:23,775 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:49:27,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:28,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:32,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:33,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:35,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:35,908 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:35,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:35,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:36,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:37,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:37,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:38,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:41,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:43,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:43,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:48,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:48,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:51,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:51,002 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:51,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:51,003 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:51,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:53,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:53,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:53,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:53,777 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:49:53,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:55,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:58,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:59,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:03,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:04,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:08,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:09,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:12,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:12,034 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:12,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:12,035 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:12,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:13,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:13,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:15,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:17,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:18,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:20,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:23,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:23,778 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:50:25,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:28,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:30,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:31,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:31,977 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:31,977 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:31,978 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:32,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:33,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:34,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:36,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:37,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:38,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:41,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:43,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:46,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:48,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:50,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:50,425 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:50,425 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:50,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:50,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:51,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:51,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:53,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:53,779 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:50:55,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:59,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:01,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:04,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:06,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:06,208 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:06,208 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:06,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:06,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:07,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:07,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:08,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:11,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:12,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:13,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:17,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:18,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:22,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:23,780 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:51:23,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:24,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:24,134 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:24,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:24,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:24,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:25,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:27,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:29,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:29,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:32,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:34,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:37,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:38,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:39,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:39,828 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:39,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:39,829 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:40,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:41,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:42,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:43,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:44,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:48,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:49,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:53,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:53,780 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:51:53,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:54,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:54,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:54,528 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:54,528 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:54,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:55,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:58,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:59,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:59,574 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:03,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:04,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:08,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:08,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:11,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:11,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:11,456 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:11,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:11,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:13,580 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:13,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:14,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:15,581 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:19,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:19,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:23,782 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:52:23,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:24,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:28,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:28,493 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:28,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:28,494 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:28,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:29,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:29,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:30,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:33,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:34,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:36,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:38,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:41,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:44,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:46,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:49,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:51,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:52,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:52,876 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:52,877 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:52,877 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:53,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:53,596 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:53,783 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:52:54,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:54,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:54,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:56,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:57,597 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:59,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:01,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:04,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:06,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:09,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:09,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:11,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:12,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:12,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:12,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:12,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:13,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:13,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:15,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:16,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:17,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:20,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:21,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:23,784 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:53:24,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:26,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:29,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:30,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:30,673 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:30,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:30,674 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:31,611 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:31,611 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:32,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:35,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:35,612 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:37,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:39,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:42,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:45,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:47,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:50,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:51,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:51,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:51,510 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:51,511 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:51,618 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:53,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:53,619 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:53,785 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:53:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:55,620 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:58,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:00,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:03,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:05,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:08,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:09,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:11,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:54:11,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:54:11,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:54:11,484 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:54:11,625 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:54:13,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:14,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:15,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:17,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:19,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:20,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:23,785 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:54:24,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:25,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:30,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:30,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:33,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:54:33,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:54:33,046 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:54:33,047 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:54:33,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:54:33,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:35,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:36,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:37,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:39,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:41,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:44,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:46,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:49,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:51,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:53,786 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:54:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:56,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:59,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:01,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:01,463 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:01,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:01,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:01,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:02,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:03,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:05,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:05,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:07,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:09,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:12,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:15,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:17,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:20,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:20,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:20,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:20,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:20,856 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:21,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:21,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:22,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:23,787 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:55:24,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:25,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:28,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:30,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:33,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:35,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:36,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:36,460 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:36,460 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:36,461 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:36,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:37,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:38,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:39,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:41,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:43,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:45,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:48,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:50,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:52,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:52,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:52,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:52,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:53,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:53,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:53,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:53,788 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:55:55,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:55,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:57,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:58,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:00,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:03,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:05,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:06,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:06,473 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:06,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:06,474 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:06,666 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:07,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:09,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:10,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:11,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:14,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:16,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:19,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:21,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:23,789 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:56:23,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:23,909 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:23,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:23,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:24,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:24,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:25,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:25,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:27,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:30,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:30,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:35,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:35,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:40,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:40,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:42,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:42,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:42,510 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:42,510 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:42,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:43,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:45,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:45,681 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:46,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:47,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:50,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:51,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:53,790 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:56:55,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:55,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:59,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:59,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:59,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:59,992 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:00,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:01,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:01,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:01,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:03,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:06,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:06,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:10,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:11,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:15,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:16,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:20,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:20,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:20,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:20,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:20,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:21,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:21,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:21,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:23,791 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:57:25,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:25,697 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:26,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:31,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:31,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:36,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:36,436 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:36,437 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:36,437 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:36,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:36,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:37,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:37,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:39,702 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:40,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:42,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:46,173 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:47,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:51,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:52,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:53,792 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:57:55,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:55,146 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:55,146 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:55,146 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:55,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:55,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:55,709 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:58,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:59,710 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:00,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:03,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:05,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:08,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:10,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:13,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:15,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:16,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:16,029 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:16,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:16,030 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:16,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:17,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:19,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:19,719 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:23,793 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:58:24,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:25,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:29,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:30,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:30,958 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:30,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:30,959 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:31,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:31,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:32,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:35,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:36,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:36,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:40,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:40,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:45,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:45,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:50,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:50,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:50,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:50,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:50,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:50,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:50,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:52,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:53,794 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:58:54,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:55,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:55,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:00,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:01,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:05,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:06,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:07,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:07,492 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:07,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:07,493 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:07,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:08,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:10,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:11,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:12,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:16,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:16,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:21,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:21,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:23,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:23,438 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:23,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:23,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:23,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:23,794 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:59:24,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:26,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:27,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:28,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:31,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:32,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:36,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:37,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:41,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:41,021 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:41,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:41,022 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:41,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:41,752 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:42,753 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:43,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:46,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:46,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:48,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:51,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:53,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:53,795 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:59:56,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:58,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:01,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:01,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:01,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:01,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:01,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:01,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:02,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:04,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:06,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:07,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:09,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:11,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:14,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:16,265 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:19,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:21,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:23,796 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:00:24,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:26,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:26,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:26,959 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:26,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:26,960 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:27,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:28,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:29,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:32,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:35,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:36,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:40,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:41,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:45,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:46,695 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:50,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:50,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:50,562 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:50,562 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:50,563 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:50,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:52,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:52,779 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:53,797 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:00:54,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:55,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:56,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:00,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:01,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:05,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:06,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:10,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:10,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:10,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:10,980 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:10,980 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:11,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:11,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:12,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:16,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:16,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:16,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:21,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:21,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:23,797 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:01:26,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:26,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:31,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:31,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:32,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:32,497 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:32,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:32,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:32,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:34,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:37,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:37,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:38,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:41,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:42,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:46,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:47,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:50,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:50,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:50,165 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:50,165 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:50,800 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:51,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:52,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:53,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:53,799 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:01:54,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:56,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:58,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:01,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:03,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:06,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:06,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:06,961 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:06,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:06,961 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:07,806 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:08,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:08,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:11,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:12,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:14,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:16,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:19,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:21,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:23,799 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:02:24,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:24,273 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:24,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:24,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:24,275 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:24,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:26,813 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:26,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:26,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:28,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:29,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:31,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:34,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:36,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:39,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:41,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:41,511 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:41,511 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:41,511 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:41,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:41,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:42,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:44,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:46,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:47,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:49,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:52,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:53,800 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:02:55,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:56,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:59,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:59,575 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:59,576 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:59,576 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:59,827 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:01,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:02,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:02,828 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:04,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:06,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:07,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:11,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:11,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:16,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:17,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:17,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:17,724 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:17,724 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:17,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:17,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:20,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:22,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:22,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:22,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:23,801 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:03:26,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:27,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:32,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:32,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:37,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:37,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:38,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:38,606 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:38,606 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:38,607 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:38,841 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:40,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:41,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:42,843 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:43,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:44,844 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:47,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:48,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:52,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:53,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:53,418 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:53,418 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:53,419 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:53,802 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:03:53,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:53,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:54,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:56,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:58,850 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:59,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:02,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:04,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:07,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:09,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:11,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:12,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:12,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:12,799 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:12,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:12,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:14,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:14,856 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:17,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:18,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:19,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:22,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:23,803 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:04:25,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:26,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:28,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:28,563 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:28,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:28,564 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:28,861 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:30,862 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:31,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:32,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:32,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:34,864 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:36,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:37,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:41,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:41,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:46,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:47,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:48,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:48,534 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:48,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:48,869 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:49,256 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:50,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:52,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:52,871 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:53,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:53,804 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:04:54,872 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:56,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:57,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:02,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:02,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:07,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:07,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:11,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:12,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:13,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:13,879 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:13,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:13,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:13,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:16,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:17,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:18,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:18,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:22,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:23,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:23,804 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:05:26,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:28,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:32,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:32,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:32,652 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:32,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:32,652 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:32,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:34,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:34,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:36,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:38,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:39,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:41,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:44,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:47,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:47,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:47,411 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:47,411 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:47,411 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:47,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:48,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:50,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:52,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:53,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:53,805 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:05:55,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:56,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:00,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:02,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:05,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:05,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:05,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:05,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:05,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:06,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:06,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:07,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:10,903 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:11,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:11,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:16,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:17,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:21,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:22,422 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:23,806 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:06:26,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:26,962 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:26,963 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:26,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:26,964 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:27,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:28,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:32,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:32,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:32,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:37,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:37,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:41,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:42,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:46,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:46,176 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:46,176 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:46,177 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:46,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:47,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:47,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:48,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:50,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:52,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:52,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:53,807 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:06:56,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:57,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:01,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:02,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:06,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:07,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:07,506 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:07,506 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:07,506 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:07,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:08,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:08,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:11,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:12,927 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:13,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:17,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:18,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:22,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:23,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:23,808 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:07:26,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:26,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:26,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:26,821 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:26,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:26,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:28,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:28,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:30,934 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:32,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:33,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:37,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:38,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:41,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:43,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:47,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:48,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:49,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:49,892 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:49,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:49,892 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:49,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:50,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:52,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:53,809 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:07:54,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:54,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:56,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:59,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:02,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:04,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:07,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:09,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:10,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:10,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:10,508 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:10,508 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:10,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:12,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:12,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:14,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:15,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:17,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:20,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:22,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:23,810 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:08:25,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:26,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:26,003 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:26,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:26,004 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:26,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:26,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:27,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:27,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:27,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:30,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:31,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:32,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:36,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:37,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:41,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:42,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:46,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:47,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:49,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:49,846 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:49,846 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:49,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:49,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:50,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:51,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:53,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:53,811 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:08:54,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:57,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:57,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:02,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:03,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:06,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:06,173 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:06,173 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:06,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:06,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:08,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:08,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:10,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:12,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:13,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:17,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:18,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:23,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:23,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:23,816 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:09:27,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:27,236 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:27,237 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:27,237 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:27,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:27,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:28,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:29,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:32,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:32,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:34,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:37,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:39,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:42,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:42,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:42,867 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:42,868 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:42,868 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:42,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:44,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:44,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:46,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:48,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:50,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:53,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:53,813 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:09:55,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:57,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:00,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:03,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:05,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:05,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:05,540 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:05,542 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:05,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:06,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:06,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:09,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:10,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:11,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:12,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:16,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:18,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:21,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:22,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:22,487 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:22,487 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:22,487 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:23,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:23,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:23,813 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:10:25,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:27,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:27,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:27,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:32,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:33,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:37,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:38,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:38,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:38,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:38,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:38,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:39,005 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:41,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:42,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:43,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:43,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:48,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:48,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:53,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:53,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:53,815 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:10:57,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:58,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:03,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:03,523 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:03,524 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:03,524 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:03,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:04,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:04,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:05,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:09,017 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:09,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:09,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:12,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:14,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:18,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:19,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:22,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:22,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:22,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:22,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:23,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:23,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:23,816 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:11:24,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:25,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:27,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:27,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:30,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:33,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:35,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:38,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:38,542 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:38,542 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:38,542 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:38,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:39,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:40,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:41,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:42,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:43,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:46,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:48,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:51,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:53,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:53,817 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:11:56,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:57,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:58,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:58,642 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:58,642 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:58,643 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:59,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:01,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:01,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:03,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:03,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:06,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:08,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:11,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:12,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:16,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:12:16,901 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:12:16,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:12:16,902 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:12:17,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:18,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:19,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:21,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:22,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:23,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:23,818 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:12:27,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:27,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:32,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:33,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:37,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:38,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:40,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:12:40,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:12:40,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:12:40,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:12:41,060 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:43,061 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:43,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:43,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:45,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:48,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:48,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:53,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:53,820 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:12:54,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:58,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:59,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:03,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:04,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:08,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:08,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:08,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:08,514 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:08,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:09,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:10,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:11,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:13,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:13,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:15,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:19,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:20,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:23,820 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:13:24,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:25,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:28,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:30,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:33,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:33,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:33,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:33,723 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:34,080 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:34,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:35,080 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:36,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:39,082 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:39,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:43,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:46,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:48,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:48,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:48,903 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:48,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:48,904 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:49,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:51,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:51,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:53,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:53,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:53,821 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:13:57,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:58,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:02,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:03,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:07,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:08,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:12,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:12,114 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:12,114 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:12,115 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:13,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:13,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:13,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:13,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:13,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:17,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:18,294 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:18,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:23,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:23,822 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:14:23,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:28,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:28,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:30,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:30,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:30,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:30,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:30,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:31,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:34,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:34,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:35,103 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:39,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:39,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:43,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:44,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:44,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:44,920 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:44,921 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:44,921 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:45,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:47,108 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:49,109 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:49,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:50,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:53,823 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:14:54,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:55,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:58,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:00,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:03,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:03,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:03,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:03,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:04,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:04,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:05,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:06,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:09,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:10,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:11,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:13,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:16,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:18,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:18,413 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:18,414 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:18,414 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:19,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:19,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:20,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:22,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:23,123 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:23,824 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:15:25,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:27,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:28,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:32,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:34,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:36,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:36,466 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:36,466 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:36,466 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:37,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:37,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:38,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:39,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:41,129 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:43,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:43,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:48,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:49,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:53,825 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:15:54,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:54,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:54,631 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:54,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:54,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:54,632 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:55,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:55,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:58,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:59,137 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:59,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:04,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:04,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:09,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:09,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:11,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:11,716 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:11,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:11,717 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:12,141 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:13,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:14,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:14,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:17,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:19,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:19,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:23,826 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:16:24,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:25,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:29,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:30,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:31,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:31,566 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:31,567 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:31,567 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:32,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:34,150 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:35,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:35,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:36,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:38,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:40,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:40,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:44,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:45,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:49,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:50,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:53,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:53,527 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:53,527 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:53,528 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:53,827 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:16:54,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:55,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:56,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:56,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:58,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:59,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:01,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:05,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:06,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:11,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:14,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:15,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:15,579 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:15,579 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:15,580 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:16,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:17,618 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:18,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:20,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:20,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:22,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:23,828 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:17:25,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:27,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:32,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:32,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:32,314 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:32,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:33,173 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:33,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:34,173 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:35,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:38,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:38,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:40,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:43,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:48,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:50,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:52,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:52,559 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:52,560 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:52,560 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:53,181 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:53,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:53,829 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:17:54,182 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:58,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:58,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:59,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:03,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:05,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:08,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:10,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:13,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:14,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:17,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:17,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:17,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:17,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:18,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:19,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:20,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:20,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:22,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:23,830 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:18:24,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:25,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:29,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:34,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:35,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:36,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:36,127 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:36,127 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:36,128 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:36,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:38,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:40,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:40,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:42,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:45,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:49,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:50,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:53,830 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:18:54,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:55,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:56,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:56,251 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:56,251 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:56,252 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:57,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:58,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:59,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:01,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:02,206 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:05,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:06,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:10,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:11,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:11,077 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:11,077 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:11,078 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:11,210 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:12,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:14,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:14,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:16,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:17,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:19,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:22,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:23,832 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:19:24,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:27,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:32,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:34,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:36,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:36,331 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:36,331 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:36,331 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:37,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:38,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:38,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:40,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:42,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:43,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:44,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:48,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:50,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:53,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:53,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:19:54,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:54,433 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:54,433 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:54,434 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:55,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:55,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:56,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:58,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:59,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:00,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:03,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:05,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:08,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:10,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:13,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:14,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:17,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:17,142 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:17,142 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:17,143 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:17,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:19,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:19,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:20,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:22,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:23,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:20:24,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:24,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:29,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:34,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:34,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:37,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:37,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:37,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:37,222 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:37,241 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:39,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:40,242 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:40,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:42,243 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:44,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:45,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:49,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:50,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:53,834 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:20:54,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:55,353 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:55,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:55,354 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:56,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:56,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:58,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:59,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:00,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:01,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:05,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:06,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:10,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:11,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:14,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:16,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:20,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:20,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:20,369 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:20,370 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:20,370 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:21,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:22,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:22,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:23,836 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:21:26,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:26,260 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:27,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:29,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:32,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:35,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:37,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:37,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:37,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:37,313 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:38,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:38,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:40,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:40,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:42,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:43,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:44,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:48,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:50,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:53,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:53,837 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:21:55,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:57,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:57,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:57,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:57,219 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:57,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:59,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:59,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:00,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:02,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:04,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:04,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:09,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:09,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:14,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:14,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:16,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:16,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:16,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:16,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:16,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:18,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:19,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:20,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:22,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:23,838 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:22:24,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:25,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:29,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:30,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:32,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:32,910 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:32,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:32,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:33,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:34,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:35,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:35,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:38,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:40,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:41,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:44,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:46,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:47,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:47,941 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:47,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:48,828 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:49,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:50,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:51,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:51,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:53,839 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:22:54,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:56,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:57,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:59,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:02,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:05,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:07,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:08,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:08,636 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:08,636 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:08,637 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:09,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:10,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:11,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:12,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:14,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:14,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:17,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:20,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:22,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:22,675 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:22,675 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:22,675 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:23,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:23,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:23,839 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:23:24,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:25,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:28,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:28,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:29,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:33,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:35,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:37,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:37,401 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:37,402 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:37,402 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:38,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:39,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:40,313 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:41,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:42,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:44,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:44,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:49,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:50,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:53,840 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:23:54,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:54,157 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:54,157 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:54,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:54,318 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:55,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:55,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:56,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:58,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:59,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:00,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:04,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:05,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:08,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:08,531 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:08,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:08,534 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:09,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:10,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:10,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:10,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:14,328 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:14,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:15,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:20,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:20,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:23,841 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:24:25,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:25,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:29,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:30,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:31,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:31,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:31,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:31,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:32,334 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:34,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:35,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:36,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:36,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:40,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:41,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:44,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:46,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:50,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:51,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:53,842 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:24:54,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:54,922 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:54,922 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:54,923 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:55,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:55,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:56,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:56,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:59,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:00,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:02,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:05,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:07,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:10,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:12,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:15,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:15,865 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:15,866 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:15,866 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:16,351 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:17,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:18,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:20,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:20,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:23,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:23,843 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:25:25,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:28,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:29,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:31,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:31,112 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:31,112 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:31,112 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:31,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:32,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:34,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:34,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:36,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:39,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:39,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:44,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:44,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:48,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:48,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:48,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:48,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:48,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:49,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:50,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:50,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:52,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:53,844 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:25:54,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:55,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:00,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:04,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:05,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:07,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:07,032 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:07,032 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:07,033 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:07,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:08,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:10,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:11,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:12,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:16,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:20,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:21,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:23,845 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:26:25,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:26,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:29,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:29,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:29,944 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:29,944 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:29,944 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:30,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:31,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:32,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:34,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:35,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:37,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:40,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:42,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:44,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:46,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:46,604 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:46,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:46,604 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:47,393 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:47,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:48,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:50,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:52,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:52,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:53,846 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:26:55,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:57,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:02,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:05,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:06,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:06,175 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:06,176 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:06,176 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:06,400 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:08,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:08,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:10,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:10,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:13,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:14,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:18,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:19,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:23,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:23,846 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:27:24,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:25,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:25,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:25,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:25,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:26,408 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:28,409 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:28,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:29,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:30,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:34,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:35,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:39,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:40,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:42,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:42,621 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:42,622 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:42,622 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:43,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:44,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:44,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:45,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:46,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:48,416 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:49,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:50,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:53,848 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:27:54,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:55,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:59,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:00,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:01,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:01,079 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:01,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:01,569 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:02,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:02,423 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:05,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:05,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:06,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:10,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:15,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:15,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:19,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:19,381 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:19,382 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:19,382 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:19,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:20,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:20,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:21,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:23,848 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:28:24,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:25,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:26,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:30,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:30,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:35,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:36,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:39,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:39,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:39,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:39,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:39,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:40,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:41,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:41,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:44,439 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:45,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:46,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:50,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:51,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:53,849 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:28:55,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:56,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:58,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:58,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:58,250 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:58,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:58,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:00,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:00,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:02,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:02,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:05,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:07,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:10,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:12,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:14,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:14,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:14,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:14,549 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:15,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:16,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:16,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:16,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:17,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:18,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:21,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:22,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:23,851 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:29:26,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:27,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:31,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:32,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:36,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:37,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:37,368 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:37,368 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:37,369 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:37,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:38,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:38,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:42,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:42,464 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:43,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:46,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:48,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:52,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:53,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:53,851 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:29:54,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:54,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:54,343 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:54,343 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:54,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:56,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:57,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:58,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:59,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:01,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:04,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:07,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:09,514 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:12,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:14,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:16,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:16,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:16,924 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:16,924 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:16,925 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:17,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:18,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:19,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:21,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:22,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:23,852 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:30:25,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:26,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:30,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:31,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:33,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:33,172 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:33,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:33,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:33,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:34,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:36,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:36,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:38,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:41,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:41,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:46,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:46,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:50,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:50,151 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:50,151 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:50,151 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:50,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:51,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:52,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:52,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:53,853 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:30:54,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:56,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:57,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:01,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:02,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:06,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:07,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:10,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:10,495 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:10,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:10,496 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:10,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:12,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:12,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:13,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:14,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:16,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:18,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:22,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:23,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:23,855 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:31:27,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:28,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:28,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:28,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:28,553 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:29,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:29,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:30,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:31,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:32,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:34,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:37,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:39,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:42,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:44,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:44,698 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:44,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:44,699 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:45,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:45,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:46,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:46,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:48,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:50,515 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:51,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:51,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:53,856 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:31:56,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:56,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:01,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:06,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:06,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:10,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:10,251 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:10,251 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:10,251 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:10,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:11,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:12,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:12,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:14,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:16,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:17,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:21,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:22,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:23,856 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:32:26,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:27,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:31,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:31,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:31,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:31,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:31,993 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:32,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:32,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:33,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:36,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:36,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:38,197 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:41,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:43,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:46,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:47,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:47,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:47,427 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:47,428 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:47,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:48,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:48,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:52,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:52,537 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:53,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:53,857 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:32:57,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:58,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:03,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:04,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:04,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:04,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:04,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:05,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:06,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:06,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:08,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:08,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:11,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:13,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:16,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:18,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:21,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:23,859 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:33:24,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:26,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:28,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:28,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:28,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:28,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:28,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:30,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:30,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:31,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:32,551 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:35,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:37,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:40,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:42,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:45,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:46,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:49,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:49,529 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:49,529 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:49,529 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:49,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:50,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:50,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:52,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:53,859 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:33:54,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:55,860 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:57,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:00,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:01,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:05,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:07,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:08,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:08,923 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:08,924 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:08,924 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:09,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:10,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:10,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:12,565 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:12,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:14,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:16,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:16,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:21,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:21,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:23,860 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:34:26,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:26,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:29,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:29,562 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:29,563 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:29,563 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:29,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:31,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:32,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:32,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:32,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:35,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:37,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:37,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:42,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:42,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:47,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:47,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:52,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:52,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:52,378 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:52,378 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:52,379 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:52,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:53,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:53,867 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:34:55,583 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:57,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:58,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:58,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:02,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:03,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:08,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:08,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:13,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:13,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:17,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:18,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:19,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:19,022 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:19,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:19,023 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:19,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:35:21,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:22,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:23,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:23,863 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:35:23,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:27,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:28,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:32,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:33,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:37,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:38,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:38,581 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:38,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:38,583 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:38,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:35:39,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:41,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:43,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:43,602 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:44,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:47,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:49,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:52,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:53,864 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:35:54,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:57,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:58,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:58,687 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:58,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:58,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:59,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:00,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:01,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:02,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:03,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:05,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:07,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:10,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:12,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:15,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:17,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:18,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:18,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:18,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:18,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:18,614 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:21,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:21,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:22,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:23,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:23,865 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:36:26,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:27,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:31,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:32,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:36,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:36,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:36,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:36,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:37,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:37,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:37,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:39,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:41,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:42,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:42,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:47,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:47,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:52,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:52,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:53,866 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:36:54,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:54,967 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:54,968 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:54,968 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:55,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:57,629 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:58,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:58,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:59,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:02,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:03,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:07,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:08,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:11,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:11,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:11,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:11,450 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:11,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:13,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:13,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:13,636 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:15,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:17,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:17,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:18,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:23,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:23,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:23,867 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:37:28,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:28,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:32,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:33,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:34,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:34,121 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:34,121 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:34,121 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:34,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:35,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:37,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:39,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:39,647 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:42,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:44,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:47,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:49,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:49,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:49,959 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:49,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:49,960 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:50,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:51,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:52,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:53,868 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:37:54,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:55,654 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:57,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:59,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:02,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:04,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:06,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:06,686 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:06,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:06,687 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:07,658 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:08,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:09,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:10,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:11,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:13,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:15,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:17,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:20,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:21,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:21,994 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:21,994 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:21,994 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:22,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:22,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:23,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:23,869 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:38:26,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:27,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:27,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:31,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:32,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:36,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:37,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:38,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:38,590 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:38,590 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:38,591 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:38,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:39,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:41,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:43,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:43,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:46,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:47,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:51,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:52,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:53,870 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:38:56,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:57,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:58,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:58,157 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:58,157 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:58,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:58,696 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:59,697 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:02,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:02,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:03,698 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:07,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:08,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:12,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:13,112 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:17,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:17,138 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:17,138 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:17,138 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:17,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:17,703 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:18,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:19,704 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:21,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:22,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:23,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:23,871 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:39:27,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:28,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:32,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:33,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:36,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:36,230 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:36,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:36,230 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:36,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:37,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:37,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:39,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:41,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:42,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:44,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:47,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:49,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:51,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:51,601 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:51,602 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:51,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:51,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:53,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:53,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:54,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:56,283 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:39:57,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:58,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:00,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:02,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:05,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:07,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:10,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:12,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:15,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:18,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:18,800 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:18,800 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:18,801 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:19,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:19,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:20,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:22,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:23,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:23,873 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:40:25,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:27,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:30,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:32,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:35,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:37,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:38,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:38,297 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:38,297 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:38,299 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:38,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:39,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:41,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:43,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:43,737 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:47,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:51,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:53,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:53,873 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:40:53,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:53,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:53,980 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:53,980 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:54,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:55,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:57,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:58,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:59,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:02,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:02,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:07,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:07,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:12,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:12,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:13,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:13,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:13,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:13,541 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:13,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:15,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:17,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:17,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:22,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:22,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:23,874 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:41:27,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:27,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:32,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:32,117 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:32,117 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:32,118 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:32,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:32,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:33,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:33,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:37,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:37,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:38,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:42,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:43,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:47,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:48,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:52,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:53,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:53,875 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:41:54,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:54,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:54,435 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:54,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:54,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:55,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:58,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:58,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:59,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:02,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:03,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:08,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:08,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:13,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:13,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:14,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:14,326 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:14,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:14,327 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:14,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:15,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:19,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:19,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:23,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:23,876 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:42:24,877 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:28,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:29,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:31,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:31,881 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:31,881 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:31,882 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:32,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:32,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:33,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:34,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:35,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:37,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:37,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:40,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:42,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:45,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:45,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:45,601 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:45,601 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:45,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:45,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:47,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:47,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:49,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:50,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:52,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:53,878 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:42:55,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:57,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:59,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:59,697 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:59,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:59,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:59,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:01,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:01,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:02,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:05,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:07,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:07,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:12,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:12,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:17,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:17,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:20,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:20,167 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:20,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:20,167 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:20,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:21,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:22,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:23,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:23,879 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:43:25,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:27,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:28,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:32,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:33,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:37,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:37,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:37,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:37,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:37,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:37,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:39,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:39,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:41,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:43,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:44,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:47,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:49,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:53,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:53,880 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:43:54,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:54,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:54,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:54,179 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:54,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:55,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:55,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:58,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:59,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:00,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:02,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:05,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:07,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:10,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:12,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:13,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:13,858 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:13,858 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:13,859 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:14,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:15,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:15,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:17,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:17,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:21,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:22,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:23,880 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:44:26,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:27,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:27,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:27,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:27,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:27,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:27,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:29,821 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:31,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:31,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:32,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:36,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:37,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:41,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:42,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:46,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:47,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:50,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:50,931 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:50,931 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:50,932 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:51,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:51,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:51,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:52,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:53,886 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:44:55,830 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:57,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:57,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:02,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:02,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:07,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:07,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:12,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:12,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:12,710 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:12,711 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:12,711 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:12,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:12,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:13,837 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:17,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:17,839 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:17,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:22,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:22,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:23,882 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:45:27,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:27,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:32,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:32,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:35,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:35,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:35,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:35,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:35,846 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:37,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:38,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:38,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:39,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:43,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:43,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:47,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:48,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:52,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:53,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:53,883 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:45:53,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:53,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:53,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:53,956 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:54,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:55,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:57,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:57,856 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:59,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:02,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:04,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:07,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:09,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:10,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:10,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:10,428 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:10,430 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:10,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:11,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:13,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:14,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:15,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:17,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:19,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:23,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:23,884 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:46:25,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:28,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:30,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:32,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:35,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:35,124 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:35,124 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:35,125 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:35,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:35,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:36,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:37,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:39,876 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:41,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:42,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:46,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:47,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:51,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:53,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:53,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:53,686 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:53,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:53,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:53,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:53,885 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:46:55,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:56,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:57,884 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:58,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:01,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:02,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:06,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:07,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:07,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:07,965 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:07,965 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:07,966 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:08,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:09,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:11,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:12,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:12,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:17,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:22,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:22,097 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:22,098 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:22,098 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:22,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:22,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:23,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:23,886 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:47:23,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:27,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:27,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:28,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:32,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:33,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:37,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:38,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:41,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:41,494 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:41,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:41,495 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:41,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:43,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:43,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:43,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:45,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:47,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:48,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:52,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:53,887 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:47:54,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:57,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:59,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:02,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:04,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:05,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:05,409 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:05,409 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:05,410 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:05,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:05,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:08,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:09,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:10,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:13,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:15,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:17,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:20,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:23,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:23,888 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:48:24,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:24,239 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:24,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:24,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:24,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:25,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:26,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:27,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:28,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:31,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:32,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:36,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:37,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:41,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:42,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:42,253 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:42,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:42,254 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:42,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:42,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:43,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:45,926 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:47,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:51,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:52,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:53,889 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:48:56,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:56,890 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:56,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:56,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:56,891 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:56,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:57,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:58,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:01,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:02,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:02,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:07,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:07,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:12,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:12,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:15,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:15,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:15,937 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:15,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:17,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:17,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:17,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:19,939 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:22,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:23,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:23,890 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:49:27,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:28,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:32,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:32,616 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:32,616 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:32,617 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:32,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:32,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:33,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:33,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:37,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:38,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:38,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:43,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:43,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:47,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:48,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:50,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:50,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:50,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:50,540 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:50,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:51,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:53,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:53,891 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:49:53,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:53,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:55,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:58,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:58,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:02,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:03,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:07,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:07,069 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:07,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:07,070 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:07,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:07,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:08,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:09,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:11,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:13,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:14,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:17,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:19,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:23,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:23,892 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:50:24,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:26,946 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:26,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:26,946 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:26,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:27,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:29,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:29,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:31,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:32,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:35,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:38,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:40,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:43,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:45,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:46,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:46,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:46,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:46,857 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:46,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:47,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:48,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:48,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:51,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:51,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:53,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:53,893 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:50:56,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:58,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:01,895 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:03,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:04,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:04,912 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:04,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:04,912 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:04,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:05,980 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:06,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:08,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:09,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:12,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:13,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:17,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:18,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:22,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:23,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:23,894 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:51:24,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:24,102 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:24,102 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:24,103 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:24,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:25,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:27,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:27,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:28,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:32,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:33,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:37,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:38,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:42,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:43,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:47,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:48,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:51,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:51,231 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:51,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:51,232 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:51,997 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:51,998 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:53,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:53,895 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:51:53,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:55,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:58,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:58,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:03,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:03,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:08,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:08,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:09,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:09,802 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:09,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:09,803 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:10,004 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:10,004 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:13,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:14,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:14,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:18,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:18,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:23,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:23,896 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:52:24,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:26,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:26,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:26,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:26,729 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:27,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:29,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:29,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:29,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:31,012 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:33,013 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:33,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:34,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:39,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:39,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:41,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:41,428 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:41,429 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:41,429 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:42,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:43,017 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:45,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:45,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:47,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:48,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:50,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:53,897 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:52:54,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:55,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:59,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:00,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:01,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:01,173 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:01,174 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:01,174 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:02,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:03,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:03,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:06,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:07,027 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:08,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:11,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:13,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:16,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:18,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:19,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:19,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:19,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:19,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:20,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:21,033 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:21,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:23,897 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:53:24,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:25,034 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:26,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:29,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:31,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:33,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:36,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:37,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:37,505 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:37,505 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:37,506 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:38,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:39,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:39,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:42,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:43,042 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:44,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:47,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:48,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:52,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:52,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:52,860 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:52,860 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:52,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:53,045 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:53,898 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:53:54,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:55,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:57,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:59,048 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:59,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:03,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:03,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:08,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:09,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:13,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:13,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:13,595 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:13,595 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:13,595 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:14,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:15,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:15,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:18,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:18,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:19,055 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:23,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:23,899 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:54:24,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:28,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:29,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:33,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:33,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:36,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:36,191 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:36,191 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:36,192 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:37,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:38,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:39,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:39,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:41,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:43,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:44,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:48,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:49,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:51,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:51,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:51,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:51,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:51,067 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:53,068 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:53,905 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:54:54,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:55,069 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:55,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:57,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:59,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:00,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:05,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:05,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:05,943 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:05,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:05,943 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:06,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:07,074 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:09,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:10,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:11,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:14,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:15,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:18,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:20,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:23,901 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:55:24,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:25,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:25,611 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:25,611 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:25,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:25,613 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:26,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:27,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:30,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:30,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:31,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:33,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:35,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:39,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:40,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:40,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:40,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:40,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:41,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:41,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:43,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:45,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:45,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:46,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:48,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:51,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:53,902 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:55:54,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:56,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:57,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:57,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:57,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:57,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:58,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:59,094 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:59,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:02,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:03,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:03,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:07,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:09,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:12,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:14,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:17,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:18,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:22,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:56:22,184 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:56:22,184 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:56:22,185 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:56:23,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:56:23,103 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:23,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:23,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:23,903 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:56:27,104 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:28,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:28,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:33,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:33,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:38,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:38,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:42,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:56:42,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:56:42,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:56:42,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:56:43,110 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:56:44,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:44,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:45,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:47,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:48,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:49,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:53,904 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:56:54,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:54,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:59,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:59,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:03,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:03,058 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:03,058 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:03,058 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:03,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:05,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:05,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:07,120 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:09,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:10,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:14,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:15,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:18,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:20,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:23,905 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:57:24,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:25,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:25,208 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:25,208 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:25,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:25,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:26,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:27,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:29,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:29,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:30,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:31,129 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:33,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:35,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:38,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:40,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:42,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:42,584 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:42,584 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:42,585 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:43,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:44,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:45,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:45,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:47,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:48,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:50,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:53,906 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:57:54,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:55,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:59,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:00,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:00,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:00,485 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:00,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:01,141 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:01,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:03,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:05,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:06,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:09,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:11,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:14,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:16,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:18,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:18,041 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:18,042 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:18,042 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:18,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:18,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:19,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:22,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:23,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:23,907 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:58:24,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:27,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:29,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:32,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:33,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:37,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:39,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:42,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:44,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:44,794 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:44,795 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:44,795 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:44,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:45,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:47,160 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:47,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:48,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:49,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:52,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:53,908 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:58:54,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:58,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:59,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:59,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:59,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:59,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:59,766 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:00,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:01,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:03,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:03,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:05,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:08,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:09,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:13,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:14,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:18,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:18,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:20,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:59:20,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:59:20,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:59:20,451 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:21,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:21,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:23,909 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:59:24,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:24,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:25,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:29,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:29,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:33,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:34,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:39,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:39,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:43,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:59:43,797 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:59:43,797 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:59:43,798 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:44,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:45,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:45,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:48,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:49,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:50,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:54,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:55,317 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:59:56,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:59,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:01,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:01,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:01,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:01,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:01,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:02,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:03,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:04,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:06,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:07,192 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:09,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:11,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:14,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:16,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:19,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:19,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:19,617 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:19,617 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:19,618 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:20,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:21,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:22,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:23,912 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:00:24,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:25,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:27,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:29,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:32,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:34,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:37,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:37,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:37,765 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:37,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:38,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:38,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:39,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:39,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:43,206 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:44,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:44,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:49,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:49,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:53,912 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:00:54,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:54,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:57,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:57,371 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:57,371 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:57,372 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:58,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:59,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:00,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:00,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:01,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:05,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:10,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:10,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:15,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:15,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:19,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:20,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:23,913 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:01:25,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:25,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:26,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:01:26,380 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:01:26,380 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:01:26,381 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:01:27,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:01:27,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:31,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:31,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:31,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:34,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:36,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:40,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:41,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:45,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:45,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:01:45,479 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:01:45,480 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:01:45,480 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:01:46,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:01:47,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:47,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:49,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:49,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:52,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:53,914 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:01:54,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:57,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:59,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:00,641 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:00,641 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:00,642 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:01,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:01,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:03,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:05,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:09,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:09,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:14,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:14,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:19,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:19,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:19,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:19,938 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:19,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:19,939 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:20,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:21,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:23,915 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:02:24,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:24,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:25,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:29,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:29,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:34,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:34,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:34,746 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:34,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:34,747 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:35,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:35,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:35,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:39,251 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:39,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:40,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:44,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:45,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:49,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:50,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:53,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:53,379 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:53,379 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:53,380 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:53,916 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:02:54,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:55,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:55,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:55,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:57,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:00,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:01,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:04,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:06,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:08,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:08,628 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:08,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:08,629 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:09,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:09,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:10,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:11,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:13,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:15,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:16,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:19,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:21,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:23,917 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:03:24,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:26,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:29,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:30,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:30,642 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:30,642 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:30,643 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:31,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:31,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:32,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:34,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:35,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:37,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:39,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:42,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:44,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:47,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:49,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:52,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:53,918 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:03:54,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:55,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:55,793 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:55,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:55,793 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:56,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:57,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:58,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:59,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:00,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:01,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:04,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:04,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:09,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:09,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:13,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:13,911 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:13,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:13,912 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:14,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:14,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:14,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:15,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:19,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:19,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:20,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:23,918 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:04:24,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:25,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:29,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:30,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:31,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:31,227 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:31,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:31,228 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:31,294 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:33,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:34,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:35,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:36,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:39,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:41,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:44,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:46,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:48,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:48,456 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:48,457 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:48,457 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:49,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:49,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:49,302 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:51,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:53,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:53,924 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:04:54,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:56,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:59,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:01,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:04,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:06,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:08,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:08,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:08,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:08,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:09,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:09,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:09,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:11,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:13,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:14,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:16,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:19,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:21,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:23,920 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:05:24,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:25,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:25,695 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:25,696 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:25,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:26,316 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:27,316 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:27,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:29,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:30,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:32,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:34,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:37,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:39,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:42,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:44,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:44,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:44,543 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:44,543 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:44,544 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:45,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:45,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:48,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:49,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:49,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:53,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:53,921 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:05:55,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:58,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:00,147 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:03,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:03,640 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:03,640 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:03,642 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:04,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:04,330 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:04,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:05,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:07,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:09,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:09,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:14,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:14,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:19,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:19,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:20,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:20,694 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:20,694 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:20,694 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:21,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:21,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:23,922 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:06:24,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:24,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:25,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:29,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:29,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:34,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:34,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:39,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:39,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:39,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:39,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:39,547 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:40,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:40,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:41,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:45,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:45,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:45,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:49,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:50,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:53,923 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:06:55,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:55,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:55,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:55,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:55,967 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:55,967 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:56,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:57,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:00,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:01,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:01,351 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:04,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:06,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:09,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:11,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:14,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:14,572 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:14,573 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:14,575 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:14,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:15,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:15,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:16,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:19,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:19,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:21,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:23,923 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:07:24,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:26,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:29,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:31,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:31,328 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:31,328 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:31,328 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:31,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:32,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:33,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:34,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:35,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:37,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:40,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:42,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:45,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:47,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:49,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:52,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:53,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:53,838 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:53,838 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:53,840 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:53,925 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:07:54,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:55,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:55,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:58,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:59,376 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:00,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:03,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:04,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:08,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:09,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:10,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:10,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:10,718 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:10,718 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:11,377 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:11,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:13,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:15,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:15,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:18,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:19,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:23,925 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:08:24,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:24,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:27,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:27,240 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:27,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:27,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:27,384 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:29,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:29,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:30,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:31,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:34,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:35,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:39,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:40,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:44,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:45,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:49,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:50,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:53,926 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:08:54,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:55,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:55,259 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:55,259 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:55,260 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:55,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:56,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:57,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:59,396 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:00,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:01,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:04,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:06,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:10,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:11,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:13,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:13,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:13,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:13,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:13,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:15,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:15,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:17,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:17,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:19,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:22,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:23,927 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:09:24,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:27,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:29,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:30,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:30,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:30,759 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:30,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:31,408 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:31,409 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:33,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:34,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:35,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:38,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:39,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:43,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:44,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:48,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:48,833 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:48,833 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:48,834 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:49,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:49,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:49,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:49,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:53,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:53,997 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:09:54,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:54,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:59,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:59,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:04,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:04,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:06,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:06,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:06,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:06,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:07,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:07,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:10,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:10,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:11,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:15,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:15,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:19,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:20,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:23,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:23,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:23,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:23,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:23,515 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:23,929 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:10:24,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:25,432 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:25,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:27,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:29,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:31,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:34,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:36,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:39,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:41,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:43,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:43,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:43,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:43,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:44,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:45,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:46,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:48,443 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:49,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:51,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:53,930 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:10:55,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:56,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:00,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:01,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:01,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:01,942 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:01,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:01,945 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:02,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:04,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:04,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:06,450 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:07,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:08,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:09,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:12,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:14,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:17,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:19,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:21,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:21,417 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:21,417 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:21,417 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:21,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:22,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:23,931 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:11:24,458 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:25,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:26,459 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:27,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:30,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:32,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:34,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:37,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:37,194 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:37,194 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:37,196 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:37,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:38,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:39,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:40,464 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:42,465 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:43,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:44,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:48,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:49,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:53,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:53,932 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:11:54,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:58,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:59,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:59,045 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:59,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:59,046 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:59,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:00,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:00,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:04,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:04,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:04,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:09,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:09,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:14,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:14,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:18,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:18,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:18,153 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:18,155 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:18,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:19,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:19,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:19,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:20,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:20,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:23,933 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:12:24,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:24,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:25,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:29,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:30,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:34,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:35,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:38,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:38,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:38,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:38,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:39,491 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:40,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:40,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:41,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:44,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:45,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:47,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:49,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:52,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:53,934 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:12:55,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:57,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:58,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:58,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:58,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:58,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:58,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:00,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:00,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:03,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:04,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:04,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:08,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:09,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:13,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:14,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:17,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:17,066 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:17,066 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:17,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:18,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:19,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:19,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:22,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:23,935 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:13:24,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:25,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:29,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:30,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:31,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:31,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:31,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:32,663 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:33,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:34,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:35,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:35,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:38,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:40,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:40,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:45,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:50,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:50,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:53,936 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:13:55,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:55,325 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:55,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:55,326 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:55,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:55,520 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:56,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:58,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:00,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:01,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:01,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:05,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:06,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:11,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:11,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:16,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:16,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:19,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:19,929 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:19,929 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:19,930 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:20,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:20,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:21,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:22,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:23,936 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:14:24,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:25,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:27,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:30,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:32,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:35,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:37,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:38,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:38,032 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:38,032 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:38,032 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:38,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:40,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:40,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:42,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:43,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:45,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:48,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:50,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:52,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:52,853 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:52,853 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:52,854 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:53,544 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:53,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:53,937 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:14:54,545 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:55,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:58,546 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:59,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:04,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:05,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:09,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:10,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:14,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:15,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:15,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:15,973 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:15,974 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:15,974 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:16,552 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:15:18,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:19,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:20,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:20,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:23,938 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:15:24,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:25,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:29,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:30,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:34,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:35,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:35,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:35,462 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:35,462 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:35,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:35,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:15:38,560 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:40,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:40,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:41,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:45,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:46,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:50,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:50,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:53,939 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:15:55,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:56,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:58,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:58,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:58,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:58,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:58,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:00,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:01,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:01,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:04,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:05,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:06,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:11,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:15,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:16,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:17,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:16:17,367 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:16:17,368 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:16:17,368 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:16:17,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:18,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:20,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:22,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:22,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:23,940 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:16:26,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:27,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:31,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:32,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:35,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:37,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:16:37,064 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:16:37,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:16:37,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:16:37,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:38,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:38,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:40,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:42,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:43,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:45,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:48,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:50,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:53,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:53,941 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:16:55,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:58,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:00,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:02,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:02,237 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:02,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:02,238 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:02,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:04,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:04,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:05,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:06,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:09,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:10,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:14,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:15,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:19,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:20,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:20,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:20,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:20,344 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:20,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:20,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:22,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:23,942 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:17:24,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:24,943 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:26,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:29,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:31,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:34,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:35,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:39,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:41,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:42,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:42,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:42,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:42,218 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:42,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:44,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:45,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:46,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:46,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:50,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:50,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:53,943 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:17:55,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:55,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:59,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:59,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:59,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:59,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:00,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:01,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:01,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:02,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:04,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:05,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:07,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:10,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:12,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:15,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:15,411 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:15,411 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:15,411 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:15,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:15,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:16,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:17,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:20,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:20,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:22,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:23,943 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:18:26,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:27,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:30,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:30,306 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:30,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:30,307 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:30,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:31,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:32,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:33,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:34,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:35,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:38,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:40,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:43,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:45,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:46,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:46,850 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:46,850 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:46,850 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:47,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:48,633 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:48,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:50,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:52,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:53,944 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:18:53,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:55,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:58,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:00,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:01,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:01,565 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:01,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:01,566 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:01,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:02,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:04,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:05,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:06,642 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:09,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:10,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:14,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:15,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:16,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:16,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:16,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:16,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:16,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:18,646 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:20,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:20,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:20,647 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:23,945 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:19:25,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:25,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:30,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:30,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:35,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:35,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:39,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:39,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:39,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:39,243 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:39,653 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:40,654 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:40,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:41,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:44,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:46,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:46,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:50,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:51,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:53,946 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:19:56,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:56,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:01,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:01,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:01,784 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:01,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:01,785 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:02,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:02,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:04,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:05,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:06,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:07,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:10,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:12,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:15,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:17,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:20,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:22,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:23,947 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:20:25,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:27,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:27,537 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:27,537 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:27,538 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:27,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:28,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:28,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:31,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:32,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:33,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:35,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:38,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:40,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:43,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:45,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:48,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:48,372 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:48,373 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:48,373 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:48,678 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:49,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:50,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:50,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:50,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:52,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:53,948 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:20:54,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:56,103 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:59,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:01,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:04,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:04,504 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:04,504 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:04,504 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:04,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:05,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:06,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:06,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:08,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:10,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:11,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:15,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:16,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:20,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:21,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:21,030 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:21,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:21,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:21,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:21,690 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:22,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:23,949 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:21:26,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:26,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:26,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:31,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:31,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:36,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:36,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:41,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:41,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:46,154 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:46,155 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:46,155 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:46,156 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:46,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:46,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:47,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:48,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:50,702 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:51,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:52,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:53,950 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:21:56,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:57,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:01,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:02,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:06,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:06,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:06,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:06,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:06,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:08,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:08,709 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:10,710 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:12,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:13,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:17,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:18,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:21,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:23,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:23,951 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:22:25,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:25,756 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:25,757 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:25,757 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:26,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:26,716 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:27,487 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:28,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:30,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:32,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:33,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:36,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:38,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:41,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:43,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:43,849 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:43,849 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:43,850 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:44,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:44,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:44,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:46,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:48,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:50,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:51,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:53,952 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:22:55,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:56,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:00,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:01,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:03,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:03,432 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:03,432 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:03,432 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:03,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:04,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:06,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:06,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:08,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:12,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:12,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:17,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:17,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:21,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:22,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:23,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:23,280 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:23,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:23,281 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:23,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:23,954 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:23:24,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:27,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:27,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:28,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:32,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:32,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:36,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:37,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:42,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:42,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:43,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:43,048 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:43,048 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:43,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:43,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:44,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:47,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:48,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:48,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:51,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:53,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:53,955 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:23:56,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:58,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:00,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:00,880 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:00,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:00,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:01,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:02,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:02,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:04,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:04,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:06,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:09,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:11,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:14,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:16,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:17,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:17,780 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:17,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:17,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:18,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:18,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:19,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:21,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:22,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:23,956 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:24:24,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:26,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:29,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:31,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:34,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:36,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:36,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:36,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:36,343 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:36,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:36,767 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:38,768 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:40,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:40,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:42,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:45,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:47,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:50,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:51,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:53,957 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:24:55,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:57,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:00,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:04,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:04,362 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:04,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:04,364 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:04,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:06,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:06,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:07,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:08,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:11,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:12,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:16,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:17,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:19,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:19,651 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:19,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:19,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:20,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:20,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:21,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:22,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:23,958 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:25:24,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:27,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:27,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:32,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:32,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:37,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:37,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:41,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:41,235 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:41,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:41,237 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:41,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:42,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:42,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:43,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:44,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:46,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:47,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:48,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:52,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:53,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:53,959 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:25:58,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:58,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:58,147 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:58,147 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:58,148 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:58,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:58,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:59,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:02,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:03,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:04,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:07,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:09,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:12,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:14,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:15,316 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:15,318 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:15,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:16,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:18,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:18,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:19,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:20,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:23,959 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:26:24,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:28,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:29,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:33,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:34,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:37,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:37,223 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:37,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:37,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:37,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:37,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:38,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:40,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:40,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:42,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:42,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:45,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:47,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:50,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:52,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:52,198 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:52,198 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:52,200 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:52,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:52,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:52,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:53,961 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:26:55,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:56,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:57,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:00,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:02,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:05,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:07,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:10,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:12,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:13,416 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:13,417 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:13,417 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:13,418 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:13,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:14,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:18,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:18,833 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:21,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:23,961 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:27:26,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:28,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:31,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:31,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:31,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:31,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:31,837 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:32,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:32,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:33,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:34,732 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 09:27:34,733 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:34,736 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,736 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,738 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,739 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:34,739 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,739 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:36,839 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:37,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:39,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:39,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:39,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:42,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:47,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:50,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:52,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:53,962 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:27:54,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:57,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:59,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:28:02,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:28:03,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 09:28:04,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive diff --git a/wandb/run-20240411_040722-rimyh9a1/logs/debug.log b/wandb/run-20240411_040722-rimyh9a1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..091a026ceaa4b66c1f3159f811bd90b79f298893 --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/logs/debug.log @@ -0,0 +1,31 @@ +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Current SDK version is 0.16.5 +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Configure stats pid to 1050 +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Loading settings from /kaggle/working/wandb/settings +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': ''} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_init.py:_log_setup():527] Logging user logs to /kaggle/working/wandb/run-20240411_040722-rimyh9a1/logs/debug.log +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_init.py:_log_setup():528] Logging internal logs to /kaggle/working/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log +2024-04-11 04:07:22,843 INFO MainThread:1050 [wandb_init.py:_jupyter_setup():473] configuring jupyter hooks +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():567] calling init triggers +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():574] wandb.init called with sweep_config: {} +config: {} +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():617] starting backend +2024-04-11 04:07:22,844 INFO MainThread:1050 [wandb_init.py:init():621] setting up manager +2024-04-11 04:07:22,847 INFO MainThread:1050 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-04-11 04:07:22,848 INFO MainThread:1050 [wandb_init.py:init():629] backend started and connected +2024-04-11 04:07:22,865 INFO MainThread:1050 [wandb_run.py:_label_probe_notebook():1299] probe notebook +2024-04-11 04:07:23,106 INFO MainThread:1050 [wandb_init.py:init():721] updated telemetry +2024-04-11 04:07:23,108 INFO MainThread:1050 [wandb_init.py:init():754] communicating run to backend with 90.0 second timeout +2024-04-11 04:07:23,235 INFO MainThread:1050 [wandb_run.py:_on_init():2344] communicating current version +2024-04-11 04:07:23,323 INFO MainThread:1050 [wandb_run.py:_on_init():2353] got version response upgrade_message: "wandb version 0.16.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-04-11 04:07:23,324 INFO MainThread:1050 [wandb_init.py:init():805] starting run threads in backend +2024-04-11 04:07:39,317 INFO MainThread:1050 [wandb_run.py:_console_start():2323] atexit reg +2024-04-11 04:07:39,317 INFO MainThread:1050 [wandb_run.py:_redirect():2178] redirect: wrap_raw +2024-04-11 04:07:39,318 INFO MainThread:1050 [wandb_run.py:_redirect():2243] Wrapping output streams. +2024-04-11 04:07:39,318 INFO MainThread:1050 [wandb_run.py:_redirect():2268] Redirects installed. +2024-04-11 04:07:39,319 INFO MainThread:1050 [wandb_init.py:init():848] run started, returning control to user process +2024-04-11 04:07:39,324 INFO MainThread:1050 [wandb_run.py:_config_callback():1347] config_cb None None {'vocab_size': 250880, 'hidden_size': 1024, 'n_layer': 24, 'n_head': 16, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'use_cache': False, 'pretraining_tp': 1, 'apply_residual_connection_post_layernorm': False, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'bos_token_id': 1, 'eos_token_id': 2, 'slow_but_exact': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['BloomForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': 3, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'bigscience/bloomz-560m', 'transformers_version': '4.39.3', 'attention_softmax_in_fp32': True, 'bias_dropout_fusion': True, 'unk_token_id': 0, 'masked_softmax_fusion': True, 'model_type': 'bloom', 'n_inner': None, 'offset_alibi': 100, 'seq_length': 2048, 'skip_bias_add': True, 'skip_bias_add_qkv': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': False, '_load_in_4bit': True, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': True, 'load_in_8bit': False}, 'output_dir': '/kaggle/working/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 3e-05, 'weight_decay': 0.001, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 5, 'max_steps': 20000, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/kaggle/working/runs/Apr11_04-07-17_321ddc84c049', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 20, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 20, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/kaggle/working/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_32bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None} diff --git a/wandb/run-20240411_040722-rimyh9a1/run-rimyh9a1.wandb b/wandb/run-20240411_040722-rimyh9a1/run-rimyh9a1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..13b270ef6579e594b1b9a9c0286e5c0da7b901ce --- /dev/null +++ b/wandb/run-20240411_040722-rimyh9a1/run-rimyh9a1.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4edaba0145fd10b6efeaabd49949a3b168e3e7c5c305959ac6c5e045e98a02c +size 2076407