End of training

README.md

@@ -0,0 +1,21 @@
+
+---
+license: creativeml-openrail-m
+base_model: runwayml/stable-diffusion-v1-5
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- text-to-image
+- diffusers
+- lora
+inference: true
+---
+    
+# LoRA text2image fine-tuning - deepgoyal19/lora5_tb
+These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the lambdalabs/pokemon-blip-captions dataset. You can find some example images in the following. 
+
+![img_0](./image_0.png)
+![img_1](./image_1.png)
+![img_2](./image_2.png)
+
+        

log/text2image-fine-tune/1686718938.2207263/events.out.tfevents.1686718938.A40-20230526-003.3793953.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2
+size 2769

log/text2image-fine-tune/1686718938.2222803/hparams.yml

@@ -0,0 +1,49 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 15
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 15
+validation_prompts: pikachu

log/text2image-fine-tune/1686719590.3468492/events.out.tfevents.1686719590.A40-20230526-003.3796603.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4a4522643499eb3044dde554a2004af16634c1fd90f19f7352288e003ec7da2
+size 2770

log/text2image-fine-tune/1686719590.3490849/hparams.yml

@@ -0,0 +1,49 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora1_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 15
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 15
+validation_prompts: pikachu

log/text2image-fine-tune/1686719831.1614823/events.out.tfevents.1686719831.A40-20230526-003.3797971.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18
+size 2770

log/text2image-fine-tune/1686719831.1635435/hparams.yml

@@ -0,0 +1,49 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora1_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 15
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 15
+validation_prompts: pikachu

log/text2image-fine-tune/1686720913.148579/events.out.tfevents.1686720913.A40-20230526-003.3802189.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e23009bcbd80acf0569ea70abd119452ccdb9a69edcc33f7038a928bdd939482
+size 2770

log/text2image-fine-tune/1686720913.1509812/hparams.yml

@@ -0,0 +1,49 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora2_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 60
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 60
+validation_prompts: pikachu

log/text2image-fine-tune/1686720979.8038175/events.out.tfevents.1686720979.A40-20230526-003.3802846.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e
+size 2770

log/text2image-fine-tune/1686720979.8056188/hparams.yml

@@ -0,0 +1,49 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora2_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 60
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 60
+validation_prompts: pikachu

log/text2image-fine-tune/1686721041.5550754/events.out.tfevents.1686721041.A40-20230526-003.3803507.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dedc3b5204b74b2151540c3347d61ee8c84006b9c95481ff2724ac0911f8848a
+size 2770

log/text2image-fine-tune/1686721041.5566664/hparams.yml

@@ -0,0 +1,49 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora2_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 60
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 60
+validation_prompts: pikachu

log/text2image-fine-tune/1686738195.981729/events.out.tfevents.1686738195.A40-20230526-003.3854615.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9ee6471aab351b1c3a319ba55daa8f7851ec66575ac7b55078914ffef40772b
+size 2713

log/text2image-fine-tune/1686738195.9854875/hparams.yml

@@ -0,0 +1,49 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora3_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 60
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 60
+validation_prompts: null

log/text2image-fine-tune/1686739744.5394647/events.out.tfevents.1686739744.A40-20230526-003.3859977.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21bc60c8432f3fdbc26a513859f4c04a18e03a56aa6e8d29a0ee1391cc4497e3
+size 2713

log/text2image-fine-tune/1686739744.541363/hparams.yml

@@ -0,0 +1,48 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora3_tensorboard
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 60
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 60

log/text2image-fine-tune/1686740628.4824169/events.out.tfevents.1686740628.A40-20230526-003.3863181.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea1e87f8a32bfaae1eafe00dfabef13887acc06b904c6e1713d9d9bc42bf28cf
+size 2704

log/text2image-fine-tune/1686740628.4842744/hparams.yml

@@ -0,0 +1,48 @@
+accelerate_device: cuda:7
+accelerate_local_process_index: 0
+accelerate_num_processes: 1
+accelerate_optimizer_step_was_skipped: false
+accelerate_process_index: 0
+accelerate_sync_gradients: true
+accelerate_use_distributed: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+checkpointing_steps: 500
+checkpoints_total_limit: null
+disable_tqdm: false
+enable_xformers_memory_efficient_attention: true
+eval_dataset_path: null
+fp16_backend: auto
+fp16_opt_level: O1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+hub_model_id: lora4_tb
+hub_token: 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
+input_perturbation: 0
+learning_rate: 1.0e-06
+logging_dir: log
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1
+max_train_samples: null
+max_train_steps: 60
+mixed_precision: fp16
+noise_offset: 1
+num_train_epochs: 1
+num_validation_images: 1
+output_dir: /home/deepanshu/lmflowdiff/Model
+push_to_hub: true
+report_to: tensorboard
+resume_from_checkpoint: latest
+save_steps: 1000
+save_total_limit: null
+scale_lr: false
+seed: 123
+snr_gamma: null
+tracker_project_name: text2image-fine-tune
+train_batch_size: 1
+use_8bit_adam: false
+validation_epochs: 60

log/text2image-fine-tune/events.out.tfevents.1686718938.A40-20230526-003.3793953.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56db31fb0d75f58bcda44e6a5b551b0cd8ff59e10469290b8395270cf6c752b0
+size 3824602

log/text2image-fine-tune/events.out.tfevents.1686719590.A40-20230526-003.3796603.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18eb5dfa7dd09aaa9476759ed5aefafacb327e25e96e38558425e983ad8e3e80
+size 3719369

log/text2image-fine-tune/events.out.tfevents.1686719831.A40-20230526-003.3797971.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed04babe7fc1d215e5f8a80dbf2296872f61b8272a19e69a838fed25dc2a0cb0
+size 3723222

log/text2image-fine-tune/events.out.tfevents.1686720913.A40-20230526-003.3802189.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2f212308d9755444b01ca57a1428f099ae463da8c42581e405453a8bcaf6018
+size 3722023

log/text2image-fine-tune/events.out.tfevents.1686720979.A40-20230526-003.3802846.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:041ff8bfe52e59c866d87f9b1dd9a37aae790fe1b92441c351b9029d15aa980b
+size 3724395

log/text2image-fine-tune/events.out.tfevents.1686721041.A40-20230526-003.3803507.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972064b9bca317a5508f4f0b55085d6cb08b5cb1cde82c684a3b2f6d0ddc232f
+size 3723251

log/text2image-fine-tune/events.out.tfevents.1686724122.A40-20230526-003.3812918.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e380cb848e0d985ce497abd616466a455fb6dfe9bceb19df38f93b72751ca76
+size 88

log/text2image-fine-tune/events.out.tfevents.1686737268.A40-20230526-003.3850576.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13f3cd093d73bedd1bf3459d90058ad191c5a67223ba2ac6c12878a3625c7aae
+size 88

log/text2image-fine-tune/events.out.tfevents.1686737891.A40-20230526-003.3852910.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f1b9adad5b002afc7c6ebddf403821b5a083ed15a0af72fabd13ba6a9722b9b
+size 88

log/text2image-fine-tune/events.out.tfevents.1686737963.A40-20230526-003.3853473.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c4bfb33e34f3eb3dd784c897abd2f3a8957807d963fb9fc6eaa0dcd083cd46d
+size 88

log/text2image-fine-tune/events.out.tfevents.1686738195.A40-20230526-003.3854615.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87549ecae170bb23048a2ca45624d8628818bda58f8dbe4ee9064a0ff43e8941
+size 1000

log/text2image-fine-tune/events.out.tfevents.1686738251.A40-20230526-003.3855176.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e3ab5c775ff52b79e327d006329d5d13313d9053300c4358034f65be5a52c88
+size 88

log/text2image-fine-tune/events.out.tfevents.1686739744.A40-20230526-003.3859977.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69cfc2f12779ca41a1cb3683d2c8d7734fb83d39d41098edbf6560a233310bb6
+size 501625

log/text2image-fine-tune/events.out.tfevents.1686740628.A40-20230526-003.3863181.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:872ff443d210926e96e45e61da68ea1da4658c0f7eb963664cc6dd31912f43df
+size 1245950

pytorch_lora_weights.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a667d4733a587ab1fd82d05132f013e57f474869c0d379c2d9e13858753f44ce
+size 3287771

wandb/settings

@@ -0,0 +1,2 @@
+[default]
+

wandb/wandb/debug-internal.log

@@ -0,0 +1,449 @@
+2023-06-14 16:09:40,206 INFO    StreamThr :3905241 [internal.py:wandb_internal():86] W&B internal server running at pid: 3905241, started at: 2023-06-14 16:09:40.205938
+2023-06-14 16:09:40,208 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status
+2023-06-14 16:09:40,211 INFO    WriterThread:3905241 [datastore.py:open_for_write():85] open: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/run-0vx95vx0.wandb
+2023-06-14 16:09:40,212 DEBUG   SenderThread:3905241 [sender.py:send():369] send: header
+2023-06-14 16:09:40,401 DEBUG   SenderThread:3905241 [sender.py:send():369] send: run
+2023-06-14 16:09:41,650 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: check_version
+2023-06-14 16:09:41,655 INFO    SenderThread:3905241 [dir_watcher.py:__init__():211] watching files in: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files
+2023-06-14 16:09:41,655 INFO    SenderThread:3905241 [sender.py:_start_run_threads():1100] run started: 0vx95vx0 with start time 1686758980.206642
+2023-06-14 16:09:41,655 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:41,655 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:41,655 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: check_version
+2023-06-14 16:09:42,656 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:09:42,670 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: run_start
+2023-06-14 16:09:42,678 DEBUG   HandlerThread:3905241 [system_info.py:__init__():31] System info init
+2023-06-14 16:09:42,678 DEBUG   HandlerThread:3905241 [system_info.py:__init__():46] System info init done
+2023-06-14 16:09:42,679 INFO    HandlerThread:3905241 [system_monitor.py:start():181] Starting system monitor
+2023-06-14 16:09:42,679 INFO    SystemMonitor:3905241 [system_monitor.py:_start():145] Starting system asset monitoring threads
+2023-06-14 16:09:42,679 INFO    HandlerThread:3905241 [system_monitor.py:probe():201] Collecting system info
+2023-06-14 16:09:42,679 INFO    SystemMonitor:3905241 [interfaces.py:start():190] Started cpu monitoring
+2023-06-14 16:09:42,681 INFO    SystemMonitor:3905241 [interfaces.py:start():190] Started disk monitoring
+2023-06-14 16:09:42,682 INFO    SystemMonitor:3905241 [interfaces.py:start():190] Started gpu monitoring
+2023-06-14 16:09:42,683 INFO    SystemMonitor:3905241 [interfaces.py:start():190] Started memory monitoring
+2023-06-14 16:09:42,683 INFO    SystemMonitor:3905241 [interfaces.py:start():190] Started network monitoring
+2023-06-14 16:09:42,767 DEBUG   HandlerThread:3905241 [system_info.py:probe():195] Probing system
+2023-06-14 16:09:42,775 DEBUG   HandlerThread:3905241 [system_info.py:_probe_git():180] Probing git
+2023-06-14 16:09:42,811 DEBUG   HandlerThread:3905241 [system_info.py:_probe_git():188] Probing git done
+2023-06-14 16:09:42,811 DEBUG   HandlerThread:3905241 [system_info.py:probe():240] Probing system done
+2023-06-14 16:09:42,812 DEBUG   HandlerThread:3905241 [system_monitor.py:probe():210] {'os': 'Linux-5.4.0-90-generic-x86_64-with-glibc2.31', 'python': '3.9.16', 'heartbeatAt': '2023-06-14T16:09:42.767191', 'startedAt': '2023-06-14T16:09:40.185854', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py', 'git': {'remote': 'https://github.com/hendrydong/LMFlow-diffusion.git', 'commit': '7eb976e6588110787f15fb1ef1b6ce7c4d84ae87'}, 'email': 'goyaldeep178@gmail.com', 'root': '/home/deepanshu/lmflowdiff', 'host': 'A40-20230526-003', 'username': 'deepanshu', 'executable': '/home/deepanshu/anaconda3/envs/deepanshu/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 3.130054687500001, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.533, 'min': 1500.0, 'max': 2800.0}, {'current': 2.322, 'min': 1500.0, 'max': 2800.0}, {'current': 3.465, 'min': 1500.0, 'max': 2800.0}, {'current': 2.612, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.856, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.46, 'min': 1500.0, 'max': 2800.0}, {'current': 2.348, 'min': 1500.0, 'max': 2800.0}, {'current': 2.382, 'min': 1500.0, 'max': 2800.0}, {'current': 2.313, 'min': 1500.0, 'max': 2800.0}, {'current': 3.161, 'min': 1500.0, 'max': 2800.0}, {'current': 3.215, 'min': 1500.0, 'max': 2800.0}, {'current': 3.212, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.363, 'min': 1500.0, 'max': 2800.0}, {'current': 2.491, 'min': 1500.0, 'max': 2800.0}, {'current': 3.406, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.818, 'min': 1500.0, 'max': 2800.0}, {'current': 3.23, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.023, 'min': 1500.0, 'max': 2800.0}, {'current': 3.395, 'min': 1500.0, 'max': 2800.0}, {'current': 3.115, 'min': 1500.0, 'max': 2800.0}, {'current': 2.718, 'min': 1500.0, 'max': 2800.0}, {'current': 2.756, 'min': 1500.0, 'max': 2800.0}, {'current': 3.473, 'min': 1500.0, 'max': 2800.0}, {'current': 2.108, 'min': 1500.0, 'max': 2800.0}, {'current': 2.274, 'min': 1500.0, 'max': 2800.0}, {'current': 1.617, 'min': 1500.0, 'max': 2800.0}, {'current': 2.259, 'min': 1500.0, 'max': 2800.0}, {'current': 3.017, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.003, 'min': 1500.0, 'max': 2800.0}, {'current': 3.0, 'min': 1500.0, 'max': 2800.0}, {'current': 3.007, 'min': 1500.0, 'max': 2800.0}, {'current': 3.019, 'min': 1500.0, 'max': 2800.0}, {'current': 3.49, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.965, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 3.478, 'min': 1500.0, 'max': 2800.0}, {'current': 2.802, 'min': 1500.0, 'max': 2800.0}, {'current': 2.961, 'min': 1500.0, 'max': 2800.0}, {'current': 2.113, 'min': 1500.0, 'max': 2800.0}, {'current': 2.343, 'min': 1500.0, 'max': 2800.0}, {'current': 2.286, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.243, 'min': 1500.0, 'max': 2800.0}, {'current': 2.902, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 1.992, 'min': 1500.0, 'max': 2800.0}, {'current': 2.51, 'min': 1500.0, 'max': 2800.0}, {'current': 2.986, 'min': 1500.0, 'max': 2800.0}, {'current': 3.222, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.281, 'min': 1500.0, 'max': 2800.0}, {'current': 3.046, 'min': 1500.0, 'max': 2800.0}, {'current': 2.506, 'min': 1500.0, 'max': 2800.0}, {'current': 2.536, 'min': 1500.0, 'max': 2800.0}, {'current': 2.544, 'min': 1500.0, 'max': 2800.0}, {'current': 3.285, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.29, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.757, 'min': 1500.0, 'max': 2800.0}, {'current': 2.531, 'min': 1500.0, 'max': 2800.0}, {'current': 2.388, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.93, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.483, 'min': 1500.0, 'max': 2800.0}, {'current': 3.476, 'min': 1500.0, 'max': 2800.0}, {'current': 3.48, 'min': 1500.0, 'max': 2800.0}, {'current': 3.073, 'min': 1500.0, 'max': 2800.0}, {'current': 3.03, 'min': 1500.0, 'max': 2800.0}, {'current': 2.99, 'min': 1500.0, 'max': 2800.0}, {'current': 1.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.027, 'min': 1500.0, 'max': 2800.0}, {'current': 3.477, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.488, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.479, 'min': 1500.0, 'max': 2800.0}, {'current': 3.108, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.469, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.227, 'min': 1500.0, 'max': 2800.0}, {'current': 3.491, 'min': 1500.0, 'max': 2800.0}, {'current': 1.884, 'min': 1500.0, 'max': 2800.0}, {'current': 3.424, 'min': 1500.0, 'max': 2800.0}, {'current': 3.119, 'min': 1500.0, 'max': 2800.0}, {'current': 3.238, 'min': 1500.0, 'max': 2800.0}, {'current': 3.121, 'min': 1500.0, 'max': 2800.0}, {'current': 3.095, 'min': 1500.0, 'max': 2800.0}], 'disk': {'total': 218.5130844116211, 'used': 49.349178314208984}, 'gpu': 'NVIDIA A40', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}], 'memory': {'total': 755.7487335205078}}
+2023-06-14 16:09:42,812 INFO    HandlerThread:3905241 [system_monitor.py:probe():211] Finished collecting system info
+2023-06-14 16:09:42,812 INFO    HandlerThread:3905241 [system_monitor.py:probe():214] Publishing system info
+2023-06-14 16:09:42,812 DEBUG   HandlerThread:3905241 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
+2023-06-14 16:09:42,813 DEBUG   HandlerThread:3905241 [system_info.py:_save_pip():67] Saving pip packages done
+2023-06-14 16:09:42,813 DEBUG   HandlerThread:3905241 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment
+2023-06-14 16:09:43,656 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml
+2023-06-14 16:09:43,657 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/requirements.txt
+2023-06-14 16:09:51,458 DEBUG   HandlerThread:3905241 [system_info.py:_save_conda():86] Saving conda packages done
+2023-06-14 16:09:51,460 INFO    HandlerThread:3905241 [system_monitor.py:probe():216] Finished publishing system info
+2023-06-14 16:09:51,466 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:09:51,466 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive
+2023-06-14 16:09:51,466 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:09:51,467 DEBUG   SenderThread:3905241 [sender.py:send():369] send: files
+2023-06-14 16:09:51,467 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now
+2023-06-14 16:09:51,482 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:09:51,484 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:09:51,659 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml
+2023-06-14 16:09:51,659 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-metadata.json
+2023-06-14 16:09:52,979 INFO    wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/gpvyb6vu-wandb-metadata.json
+2023-06-14 16:09:53,018 DEBUG   SenderThread:3905241 [sender.py:send():369] send: telemetry
+2023-06-14 16:09:53,018 DEBUG   SenderThread:3905241 [sender.py:send():369] send: config
+2023-06-14 16:09:53,660 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:09:55,661 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:09:55,844 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:09:55,994 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:56,352 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:56,353 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:56,353 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:56,353 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:56,617 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:56,617 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:56,617 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:56,618 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:56,661 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:09:56,917 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:56,917 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:56,917 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:56,918 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:57,332 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:57,333 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:57,333 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:57,334 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:57,662 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:09:57,663 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:09:57,870 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:57,872 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:57,872 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:57,872 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:58,146 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:58,152 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:58,152 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:58,153 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:58,515 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:58,516 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:58,516 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:58,517 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:58,662 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:09:59,135 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:59,144 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:59,144 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:59,145 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:09:59,663 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:09:59,664 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:09:59,987 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:09:59,991 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:09:59,992 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:09:59,992 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:00,580 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:00,580 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:00,581 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:00,581 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:00,664 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:01,111 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:01,117 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:01,117 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:01,117 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:01,118 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:01,405 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:01,405 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:01,405 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:01,405 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:01,664 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:01,665 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:01,668 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:01,669 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:01,669 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:01,669 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:01,949 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:01,950 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:01,950 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:01,950 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:02,184 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:02,185 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:02,186 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:02,186 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:02,440 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:02,441 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:02,441 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:02,441 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:02,666 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:02,686 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:02,688 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:02,688 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:02,688 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:02,972 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:02,973 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:02,973 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:02,973 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:03,251 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:03,252 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:03,252 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:03,252 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:03,527 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:03,528 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:03,528 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:03,528 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:03,666 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:03,666 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:03,812 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:03,813 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:03,813 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:03,813 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:04,089 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:04,090 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:04,090 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:04,091 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:04,389 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:04,389 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:04,390 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:04,390 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:04,636 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:04,636 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:04,636 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:04,637 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:04,666 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:04,855 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:04,856 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:04,856 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:04,857 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:05,084 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:05,085 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:05,085 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:05,085 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:05,317 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:05,318 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:05,318 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:05,318 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:05,561 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:05,562 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:05,562 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:05,562 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:05,667 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:05,667 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:05,797 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:05,797 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:05,797 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:05,798 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:06,053 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:06,054 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:06,054 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:06,054 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:06,364 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:06,365 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:06,365 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:06,365 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:06,366 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:06,473 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:10:06,474 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:10:06,667 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:06,696 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:06,862 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:06,862 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:06,863 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:07,012 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:07,013 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:07,013 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:07,013 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:07,314 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:07,314 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:07,315 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:07,315 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:07,559 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:07,559 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:07,560 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:07,560 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:07,668 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:07,668 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:07,903 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:07,904 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:07,904 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:07,904 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:08,145 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:08,146 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:08,146 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:08,147 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:08,407 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:08,408 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:08,408 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:08,408 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:08,636 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:08,636 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:08,637 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:08,637 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:08,668 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:08,977 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:08,977 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:08,978 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:08,978 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:09,193 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:09,194 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:09,194 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:09,194 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:09,508 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:09,508 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:09,508 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:09,509 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:09,669 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:09,669 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:09,800 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:09,801 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:09,801 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:09,802 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:10,151 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:10,152 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:10,153 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:10,153 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:10,411 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:10,412 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:10,412 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:10,412 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:10,669 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:10,822 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:10,822 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:10,823 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:10,823 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:11,156 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:11,157 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:11,157 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:11,157 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:11,485 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:11,486 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:11,670 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:11,670 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:11,743 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:11,894 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:11,894 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:11,895 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:11,895 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:11,896 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:11,897 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:12,044 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:12,045 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:12,045 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:12,045 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:12,291 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:12,292 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:12,292 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:12,292 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:12,670 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml
+2023-06-14 16:10:12,671 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:12,672 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:12,674 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:12,675 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:12,675 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:13,059 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:13,061 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:13,061 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:13,062 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:13,638 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:13,646 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:13,652 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:13,655 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:13,672 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:13,672 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:14,246 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:14,249 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:14,249 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:14,250 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:14,676 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:14,834 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:14,836 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:14,837 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:14,837 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:15,545 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:15,549 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:15,549 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:15,550 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:15,677 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:15,677 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:15,864 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:15,866 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:15,866 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:15,866 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:16,155 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:16,156 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:16,157 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:16,157 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:16,677 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:17,157 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:17,677 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:19,678 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:21,488 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:10:21,488 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:10:21,679 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:22,324 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:23,680 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:25,681 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:27,561 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:27,682 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:28,590 DEBUG   SenderThread:3905241 [sender.py:send():369] send: files
+2023-06-14 16:10:28,592 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_e9c4a470ad900801f7de.png with policy now
+2023-06-14 16:10:28,599 DEBUG   SenderThread:3905241 [sender.py:send():369] send: files
+2023-06-14 16:10:28,607 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_2ff5d73691ebb72c877d.png with policy now
+2023-06-14 16:10:28,633 DEBUG   SenderThread:3905241 [sender.py:send():369] send: files
+2023-06-14 16:10:28,642 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_b3033a9fe6335a935570.png with policy now
+2023-06-14 16:10:28,649 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
+2023-06-14 16:10:28,664 DEBUG   SenderThread:3905241 [sender.py:send():369] send: history
+2023-06-14 16:10:28,664 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
+2023-06-14 16:10:28,666 INFO    SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
+2023-06-14 16:10:28,682 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
+2023-06-14 16:10:28,683 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png
+2023-06-14 16:10:28,683 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png
+2023-06-14 16:10:28,683 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png
+2023-06-14 16:10:28,683 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images
+2023-06-14 16:10:28,683 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media
+2023-06-14 16:10:30,686 INFO    wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/25n7cl81-media/images/validation_60_2ff5d73691ebb72c877d.png
+2023-06-14 16:10:31,331 INFO    wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/34c9bzlj-media/images/validation_60_e9c4a470ad900801f7de.png
+2023-06-14 16:10:31,684 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:32,001 INFO    wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ew6x2jre-media/images/validation_60_2ff5d73691ebb72c877d.png
+2023-06-14 16:10:32,323 INFO    wandb-upload_2:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5utpuer6-media/images/validation_60_b3033a9fe6335a935570.png
+2023-06-14 16:10:33,058 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:33,739 INFO    wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5zhjwo2z-media/images/validation_60_b3033a9fe6335a935570.png
+2023-06-14 16:10:33,806 INFO    wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ykn8lu3m-media/images/validation_60_e9c4a470ad900801f7de.png
+2023-06-14 16:10:35,685 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:36,491 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:10:36,493 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:10:38,156 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:39,687 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:41,688 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:42,683 DEBUG   SystemMonitor:3905241 [system_monitor.py:_start():159] Starting system metrics aggregation loop
+2023-06-14 16:10:42,686 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:10:43,389 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:43,694 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:45,690 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:47,691 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:48,724 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:51,489 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:10:51,489 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:10:51,692 INFO    Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
+2023-06-14 16:10:54,194 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:10:59,195 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:04,196 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:06,488 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:11:06,489 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:11:09,299 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:12,688 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:11:14,690 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:19,700 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:21,489 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:11:21,489 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:11:24,844 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:29,845 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:34,847 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:36,489 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:11:36,490 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:11:39,904 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:42,691 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:11:45,693 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:50,693 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:11:51,490 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:11:51,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:11:55,795 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:00,796 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:05,797 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:06,490 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:12:06,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:12:11,782 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:12,693 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:12:17,694 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:21,490 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:12:21,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:12:22,832 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:27,835 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:32,833 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:36,490 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:12:36,492 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:12:41,149 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:42,695 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:12:46,696 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:51,490 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:12:51,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:12:51,947 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:12:56,955 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:01,956 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:06,490 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:13:06,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:13:07,794 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:12,697 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:13:13,707 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:18,708 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:21,491 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:13:21,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:13:23,774 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:28,779 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:33,776 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:36,491 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:13:36,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:13:41,492 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive
+2023-06-14 16:13:42,393 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:42,701 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:13:47,703 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:51,490 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:13:51,491 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:13:52,874 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:13:57,874 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:14:02,875 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:14:06,494 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
+2023-06-14 16:14:06,494 DEBUG   SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
+2023-06-14 16:14:08,110 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
+2023-06-14 16:14:12,702 DEBUG   SenderThread:3905241 [sender.py:send():369] send: stats
+2023-06-14 16:14:13,703 DEBUG   HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report

wandb/wandb/debug.log

@@ -0,0 +1,30 @@
+2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Configure stats pid to 3904922
+2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/.config/wandb/settings
+2023-06-14 16:09:40,187 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/lmflowdiff/Model/wandb/wandb/settings
+2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CACHE_DIR
+2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CONFIG_DIR
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/deepanshu/lmflowdiff/Model/wandb'}
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-06-14 16:09:40,199 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/deepanshu/LMFlow-diffusion-main/examples/finetune.py
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py'}
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:_log_setup():507] Logging user logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug.log
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:_log_setup():508] Logging internal logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug-internal.log
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():547] calling init triggers
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():596] starting backend
+2023-06-14 16:09:40,199 INFO    MainThread:3904922 [wandb_init.py:init():600] setting up manager
+2023-06-14 16:09:40,201 INFO    MainThread:3904922 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-06-14 16:09:40,206 INFO    MainThread:3904922 [wandb_init.py:init():606] backend started and connected
+2023-06-14 16:09:40,210 INFO    MainThread:3904922 [wandb_init.py:init():703] updated telemetry
+2023-06-14 16:09:40,398 INFO    MainThread:3904922 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
+2023-06-14 16:09:41,650 INFO    MainThread:3904922 [wandb_run.py:_on_init():2176] communicating current version
+2023-06-14 16:09:42,662 INFO    MainThread:3904922 [wandb_run.py:_on_init():2185] got version response 
+2023-06-14 16:09:42,662 INFO    MainThread:3904922 [wandb_init.py:init():787] starting run threads in backend
+2023-06-14 16:09:51,480 INFO    MainThread:3904922 [wandb_run.py:_console_start():2155] atexit reg
+2023-06-14 16:09:51,481 INFO    MainThread:3904922 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
+2023-06-14 16:09:51,482 INFO    MainThread:3904922 [wandb_run.py:_redirect():2075] Wrapping output streams.
+2023-06-14 16:09:51,482 INFO    MainThread:3904922 [wandb_run.py:_redirect():2100] Redirects installed.
+2023-06-14 16:09:51,483 INFO    MainThread:3904922 [wandb_init.py:init():828] run started, returning control to user process
+2023-06-14 16:09:51,483 INFO    MainThread:3904922 [wandb_run.py:_config_callback():1283] config_cb None None {'num_validation_images': 1, 'validation_epochs': 60, 'max_train_samples': None, 'output_dir': '/home/deepanshu/lmflowdiff/Model', 'seed': 123, 'train_batch_size': 1, 'num_train_epochs': 1, 'max_train_steps': 60, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1, 'push_to_hub': True, 'hub_token': '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', 'hub_model_id': 'lora5_tb', 'logging_dir': 'log', 'mixed_precision': 'fp16', 'resume_from_checkpoint': 'latest', 'fp16_backend': 'auto', 'fp16_opt_level': 'O1', 'disable_tqdm': False, 'save_steps': 1000, 'save_total_limit': None, 'eval_dataset_path': None, 'tracker_project_name': 'text2image-fine-tune', 'checkpoints_total_limit': None, 'enable_xformers_memory_efficient_attention': True, 'noise_offset': 1, 'input_perturbation': 0, 'checkpointing_steps': 500, 'report_to': 'wandb', 'accelerate_device': 'cuda:1', 'accelerate_local_process_index': 0, 'accelerate_num_processes': 1, 'accelerate_optimizer_step_was_skipped': False, 'accelerate_process_index': 0, 'accelerate_sync_gradients': True, 'accelerate_use_distributed': False}

wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml

@@ -0,0 +1,191 @@
+name: deepanshu
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - asttokens=2.2.1=pyhd8ed1ab_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0
+  - ca-certificates=2023.5.7=hbcca054_0
+  - debugpy=1.5.1=py39h295c915_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - ipykernel=6.15.0=pyh210e3f2_0
+  - ipython=8.13.2=pyh41d4057_0
+  - jedi=0.18.2=pyhd8ed1ab_0
+  - jupyter_client=7.0.6=pyhd8ed1ab_0
+  - jupyter_core=5.3.0=py39hf3d152e_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - ncurses=6.4=h6a678d5_0
+  - nest-asyncio=1.5.6=pyhd8ed1ab_0
+  - openssl=1.1.1t=h7f8727e_0
+  - packaging=23.1=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pip=23.0.1=py39h06a4308_0
+  - platformdirs=3.5.1=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.38=pyha770c72_0
+  - prompt_toolkit=3.0.38=hd8ed1ab_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pygments=2.15.1=pyhd8ed1ab_0
+  - python=3.9.16=h7a1cb2a_2
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.9=2_cp39
+  - pyzmq=19.0.2=py39hb69f2a1_2
+  - readline=8.2=h5eee18b_0
+  - setuptools=66.0.0=py39h06a4308_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - tk=8.6.12=h1ccaba5_0
+  - tornado=6.1=py39hb9d737c_3
+  - traitlets=5.9.0=pyhd8ed1ab_0
+  - typing_extensions=4.6.2=pyha770c72_0
+  - wcwidth=0.2.6=pyhd8ed1ab_0
+  - wheel=0.38.4=py39h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - zeromq=4.3.4=h9c3ff4c_1
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - absl-py==1.4.0
+      - accelerate==0.19.0
+      - aiofiles==23.1.0
+      - aiohttp==3.8.4
+      - aiosignal==1.3.1
+      - altair==5.0.1
+      - anyio==3.7.0
+      - appdirs==1.4.4
+      - async-timeout==4.0.2
+      - attrs==23.1.0
+      - bitsandbytes==0.39.0
+      - cachetools==5.3.1
+      - certifi==2022.12.7
+      - charset-normalizer==2.1.1
+      - click==8.1.3
+      - clip==1.0
+      - cmake==3.25.0
+      - contourpy==1.0.7
+      - cycler==0.11.0
+      - datasets==2.12.0
+      - deepspeed==0.9.2
+      - diffusers==0.16.1
+      - dill==0.3.6
+      - docker-pycreds==0.4.0
+      - exceptiongroup==1.1.1
+      - fastapi==0.95.2
+      - ffmpy==0.3.0
+      - filelock==3.9.0
+      - fonttools==4.39.4
+      - frozenlist==1.3.3
+      - fsspec==2023.5.0
+      - ftfy==6.1.1
+      - gitdb==4.0.10
+      - gitpython==3.1.31
+      - google-auth==2.20.0
+      - google-auth-oauthlib==1.0.0
+      - gradio==3.32.0
+      - gradio-client==0.2.5
+      - grpcio==1.54.2
+      - h11==0.14.0
+      - hjson==3.1.0
+      - httpcore==0.17.2
+      - httpx==0.24.1
+      - huggingface-hub==0.14.1
+      - idna==3.4
+      - importlib-metadata==6.6.0
+      - importlib-resources==5.12.0
+      - jinja2==3.1.2
+      - jsonschema==4.17.3
+      - kiwisolver==1.4.4
+      - linkify-it-py==2.0.2
+      - lit==15.0.7
+      - markdown==3.4.3
+      - markdown-it-py==2.2.0
+      - markupsafe==2.1.2
+      - matplotlib==3.7.1
+      - mdit-py-plugins==0.3.3
+      - mdurl==0.1.2
+      - mpmath==1.2.1
+      - multidict==6.0.4
+      - multiprocess==0.70.14
+      - mxnet==1.9.1
+      - mypy-extensions==1.0.0
+      - natsort==8.3.1
+      - networkx==3.0
+      - ninja==1.11.1
+      - numpy==1.24.1
+      - oauthlib==3.2.2
+      - open-clip-torch==2.20.0
+      - orjson==3.8.14
+      - pandas==2.0.1
+      - pathtools==0.1.2
+      - pillow==9.3.0
+      - plotly==5.15.0
+      - protobuf==3.20.3
+      - psutil==5.9.5
+      - py-cpuinfo==9.0.0
+      - pyarrow==12.0.0
+      - pyasn1==0.5.0
+      - pyasn1-modules==0.3.0
+      - pydantic==1.10.8
+      - pydub==0.25.1
+      - pyparsing==3.0.9
+      - pyre-extensions==0.0.29
+      - pyrsistent==0.19.3
+      - python-graphviz==0.8.4
+      - python-multipart==0.0.6
+      - pytz==2023.3
+      - pyyaml==6.0
+      - regex==2023.5.5
+      - requests==2.28.1
+      - requests-oauthlib==1.3.1
+      - responses==0.18.0
+      - rsa==4.9
+      - safetensors==0.3.1
+      - scipy==1.10.1
+      - semantic-version==2.10.0
+      - sentencepiece==0.1.99
+      - sentry-sdk==1.25.1
+      - setproctitle==1.3.2
+      - smmap==5.0.0
+      - sniffio==1.3.0
+      - starlette==0.27.0
+      - sympy==1.11.1
+      - tenacity==8.2.2
+      - tensorboard==2.13.0
+      - tensorboard-data-server==0.7.1
+      - timm==0.9.2
+      - tokenizers==0.13.3
+      - toolz==0.12.0
+      - torch==2.0.1+cu118
+      - torchaudio==2.0.2+cu118
+      - torchvision==0.15.2+cu118
+      - tqdm==4.65.0
+      - transformers==4.29.2
+      - triton==2.0.0
+      - typing-extensions==4.4.0
+      - typing-inspect==0.9.0
+      - tzdata==2023.3
+      - uc-micro-py==1.0.2
+      - urllib3==1.26.13
+      - uvicorn==0.22.0
+      - wandb==0.15.4
+      - websockets==11.0.3
+      - werkzeug==2.3.6
+      - xformers==0.0.20
+      - xxhash==3.2.0
+      - yarl==1.9.2
+      - zipp==3.15.0
+prefix: /home/deepanshu/anaconda3/envs/deepanshu

wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml

@@ -0,0 +1,182 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.9.16
+    cli_version: 0.15.4
+    framework: huggingface
+    huggingface_version: 4.29.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1686758980.206642
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      3:
+      - 23
+      4: 3.9.16
+      5: 0.15.4
+      6: 4.29.2
+      8:
+      - 5
+num_validation_images:
+  desc: null
+  value: 1
+validation_epochs:
+  desc: null
+  value: 60
+max_train_samples:
+  desc: null
+  value: null
+output_dir:
+  desc: null
+  value: /home/deepanshu/lmflowdiff/Model
+seed:
+  desc: null
+  value: 123
+train_batch_size:
+  desc: null
+  value: 1
+num_train_epochs:
+  desc: null
+  value: 1
+max_train_steps:
+  desc: null
+  value: 60
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+gradient_checkpointing:
+  desc: null
+  value: true
+learning_rate:
+  desc: null
+  value: 1.0e-06
+scale_lr:
+  desc: null
+  value: false
+lr_scheduler:
+  desc: null
+  value: constant
+lr_warmup_steps:
+  desc: null
+  value: 0
+snr_gamma:
+  desc: null
+  value: null
+use_8bit_adam:
+  desc: null
+  value: false
+allow_tf32:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_weight_decay:
+  desc: null
+  value: 0.01
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1
+push_to_hub:
+  desc: null
+  value: true
+hub_token:
+  desc: null
+  value: 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
+hub_model_id:
+  desc: null
+  value: lora5_tb
+logging_dir:
+  desc: null
+  value: log
+mixed_precision:
+  desc: null
+  value: fp16
+resume_from_checkpoint:
+  desc: null
+  value: latest
+fp16_backend:
+  desc: null
+  value: auto
+fp16_opt_level:
+  desc: null
+  value: O1
+disable_tqdm:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 1000
+save_total_limit:
+  desc: null
+  value: null
+eval_dataset_path:
+  desc: null
+  value: null
+tracker_project_name:
+  desc: null
+  value: text2image-fine-tune
+checkpoints_total_limit:
+  desc: null
+  value: null
+enable_xformers_memory_efficient_attention:
+  desc: null
+  value: true
+noise_offset:
+  desc: null
+  value: 1
+input_perturbation:
+  desc: null
+  value: 0
+checkpointing_steps:
+  desc: null
+  value: 500
+report_to:
+  desc: null
+  value: wandb
+accelerate_device:
+  desc: null
+  value: cuda:1
+accelerate_local_process_index:
+  desc: null
+  value: 0
+accelerate_num_processes:
+  desc: null
+  value: 1
+accelerate_optimizer_step_was_skipped:
+  desc: null
+  value: false
+accelerate_process_index:
+  desc: null
+  value: 0
+accelerate_sync_gradients:
+  desc: null
+  value: true
+accelerate_use_distributed:
+  desc: null
+  value: false

wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log

@@ -0,0 +1,63 @@
+06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - ***** Running training *****
+06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Num examples = 833
+06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Num Epochs = 1
+06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Instantaneous batch size per device = 1
+06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Total train batch size (w. parallel, distributed & accumulation) = 1
+06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Gradient Accumulation steps = 1
+06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner -   Total optimization steps = 60
+Steps:   0%|                                                                                                                                                | 0/60 [00:00<?, ?it/s]
+Steps:   0%|                                                                                                                                                | 0/60 [00:00<?, ?it/s]/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/xformers/ops/fmha/flash.py:339: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
+  and inp.query.storage().data_ptr() == inp.key.storage().data_ptr()
+
+
+
+
+
+
+
+
+
+Steps: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:24<00:00,  2.31it/s, lr=1e-6, step_loss=0.438]06/14/2023 16:10:16 - INFO - lmflow_aigen.models.diffusion_model - Running validation...
+ Generating 1 images with prompt: ['pikachu', 'bulbasaur', 'charmandar'].
+{'requires_safety_checker'} was not found in config. Values will be initialized to default values.
+`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden.
+{'prediction_type'} was not found in config. Values will be initialized to default values.
+{'scaling_factor'} was not found in config. Values will be initialized to default values.
+Potential NSFW content was detected in one or more images. A black image will be returned instead. Try again with a different prompt and/or seed.██| 20/20 [00:02<00:00,  9.69it/s]
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20/20 [00:01<00:00, 11.02it/s]
+
+
+Model weights saved in /home/deepanshu/lmflowdiff/Model/pytorch_lora_weights.bin███████████████████████████████████████████████████████████▉       | 19/20 [00:02<00:00,  8.16it/s]
+events.out.tfevents.1686720913.A40-20230526-003.3802189.1: 100%|██████████████████████████████████████████████████████████████████████████████| 2.77k/2.77k [00:01<00:00, 2.69kB/s]
+events.out.tfevents.1686719590.A40-20230526-003.3796603.1: 100%|██████████████████████████████████████████████████████████████████████████████| 2.77k/2.77k [00:01<00:00, 2.50kB/s]
+events.out.tfevents.1686719590.A40-20230526-003.3796603.1: 100%|██████████████████████████████████████████████████████████████████████████████| 2.77k/2.77k [00:00<00:00, 3.62kB/s]
+events.out.tfevents.1686719831.A40-20230526-003.3797971.1:   0%|                                                                                       | 0.00/2.77k [00:00<?, ?B/s]
+Upload 24 LFS files:   0%|                                                                                                                                  | 0/24 [00:00<?, ?it/s]
+events.out.tfevents.1686720979.A40-20230526-003.3802846.1:   0%|                                                                                       | 0.00/2.77k [00:00<?, ?B/s]
+'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
+06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - 'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
+Retrying in 1s [Retry 1/5].
+06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5].
+'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
+06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - 'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
+'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
+06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - 'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
+Retrying in 1s [Retry 1/5].
+06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5].
+Retrying in 1s [Retry 1/5].
+06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5].
+events.out.tfevents.1686718938.A40-20230526-003.3793953.1: 100%|████████████████████████████████████████████████████████████████████████████████| 2.77k/2.77k [00:07<00:00, 388B/s]
+events.out.tfevents.1686719831.A40-20230526-003.3797971.0:   0%|                                                                                       | 0.00/3.72M [00:00<?, ?B/s]
+events.out.tfevents.1686724122.A40-20230526-003.3812918.0: 100%|██████████████████████████████████████████████████████████████████████████████████| 88.0/88.0 [00:00<00:00, 307B/s]
+events.out.tfevents.1686724122.A40-20230526-003.3812918.0:   0%|                                                                                        | 0.00/88.0 [00:00<?, ?B/s]
+events.out.tfevents.1686719831.A40-20230526-003.3797971.1: 100%|████████████████████████████████████████████████████████████████████████████████| 2.77k/2.77k [00:09<00:00, 291B/s]
+events.out.tfevents.1686720913.A40-20230526-003.3802189.0:  71%|███████████████████████████████████████████████████████                       | 2.63M/3.72M [00:00<00:00, 10.3MB/s]
+events.out.tfevents.1686737891.A40-20230526-003.3852910.0: 100%|█████████████████████████████████��████████████████████████████████████████████████| 88.0/88.0 [00:00<00:00, 319B/s]
+events.out.tfevents.1686737963.A40-20230526-003.3853473.0: 100%|██████████████████████████████████████████████████████████████████████████████████| 88.0/88.0 [00:00<00:00, 321B/s]
+events.out.tfevents.1686738195.A40-20230526-003.3854615.0: 100%|██████████████████████████████████████████████████████████████████████████████| 1.00k/1.00k [00:00<00:00, 3.53kB/s]
+events.out.tfevents.1686737963.A40-20230526-003.3853473.0:   0%|                                                                                        | 0.00/88.0 [00:00<?, ?B/s]
+events.out.tfevents.1686738251.A40-20230526-003.3855176.0:   0%|                                                                                        | 0.00/88.0 [00:00<?, ?B/s]
+pytorch_lora_weights.bin: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3.29M/3.29M [00:00<00:00, 3.32MB/s]
+events.out.tfevents.1686740628.A40-20230526-003.3863181.0: 100%|███████████████████████████████████████████████████████████████████████████████| 1.25M/1.25M [00:01<00:00, 898kB/s]
+events.out.tfevents.1686740628.A40-20230526-003.3863181.0:   0%|                                                                                       | 0.00/1.25M [00:00<?, ?B/s]
+