| Running 1 job | |
| { | |
| "type": "concept_slider", | |
| "training_folder": "/app/ai-toolkit/output", | |
| "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", | |
| "device": "cuda", | |
| "trigger_word": null, | |
| "performance_log_every": 10, | |
| "network": { | |
| "type": "lora", | |
| "linear": 4, | |
| "linear_alpha": 4, | |
| "conv": 16, | |
| "conv_alpha": 16, | |
| "lokr_full_rank": true, | |
| "lokr_factor": -1, | |
| "network_kwargs": { | |
| "ignore_if_contains": [] | |
| } | |
| }, | |
| "save": { | |
| "dtype": "bf16", | |
| "save_every": 25, | |
| "max_step_saves_to_keep": 4, | |
| "save_format": "diffusers", | |
| "push_to_hub": false | |
| }, | |
| "datasets": [ | |
| { | |
| "folder_path": "/app/ai-toolkit/datasets/the_cosmic_drift_slider", | |
| "control_path": null, | |
| "mask_path": null, | |
| "mask_min_value": 0.1, | |
| "default_caption": "", | |
| "caption_ext": "txt", | |
| "caption_dropout_rate": 0.05, | |
| "cache_latents_to_disk": false, | |
| "is_reg": false, | |
| "network_weight": 1, | |
| "resolution": [ | |
| 512 | |
| ], | |
| "controls": [], | |
| "shrink_video_to_frames": true, | |
| "num_frames": 1, | |
| "do_i2v": true, | |
| "flip_x": true, | |
| "flip_y": false | |
| } | |
| ], | |
| "train": { | |
| "batch_size": 4, | |
| "bypass_guidance_embedding": false, | |
| "steps": 300, | |
| "gradient_accumulation": 1, | |
| "train_unet": true, | |
| "train_text_encoder": false, | |
| "gradient_checkpointing": true, | |
| "noise_scheduler": "flowmatch", | |
| "optimizer": "adamw8bit", | |
| "timestep_type": "weighted", | |
| "content_or_style": "balanced", | |
| "optimizer_params": { | |
| "weight_decay": 0.0001 | |
| }, | |
| "unload_text_encoder": true, | |
| "cache_text_embeddings": false, | |
| "lr": 0.001, | |
| "ema_config": { | |
| "use_ema": false, | |
| "ema_decay": 0.99 | |
| }, | |
| "skip_first_sample": false, | |
| "force_first_sample": false, | |
| "disable_sampling": false, | |
| "dtype": "bf16", | |
| "diff_output_preservation": false, | |
| "diff_output_preservation_multiplier": 1, | |
| "diff_output_preservation_class": "person", | |
| "switch_boundary_every": 1 | |
| }, | |
| "model": { | |
| "name_or_path": "Qwen/Qwen-Image", | |
| "quantize": true, | |
| "qtype": "uint3|ostris/accuracy_recovery_adapters/qwen_image_torchao_uint3.safetensors", | |
| "quantize_te": true, | |
| "qtype_te": "qfloat8", | |
| "arch": "qwen_image", | |
| "low_vram": true, | |
| "model_kwargs": {} | |
| }, | |
| "sample": { | |
| "sampler": "flowmatch", | |
| "sample_every": 250, | |
| "width": 1024, | |
| "height": 1024, | |
| "samples": [ | |
| { | |
| "prompt": "A medium, eye-level shot of a small group of friends sitting on the grass in a park, chatting on a sunny afternoon. They are wearing simple t-shirts and jeans.", | |
| "network_multiplier": "-2.0", | |
| "seed": 42 | |
| }, | |
| { | |
| "prompt": "A medium, eye-level shot of a small group of friends sitting on the grass in a park, chatting on a sunny afternoon. They are wearing simple t-shirts and jeans.", | |
| "network_multiplier": "-1.0", | |
| "seed": 42 | |
| }, | |
| { | |
| "prompt": "A medium, eye-level shot of a small group of friends sitting on the grass in a park, chatting on a sunny afternoon. They are wearing simple t-shirts and jeans.", | |
| "network_multiplier": "-0.5", | |
| "seed": 42 | |
| }, | |
| { | |
| "prompt": "A medium, eye-level shot of a small group of friends sitting on the grass in a park, chatting on a sunny afternoon. They are wearing simple t-shirts and jeans.", | |
| "network_multiplier": "0.0", | |
| "seed": 42 | |
| }, | |
| { | |
| "prompt": "A medium, eye-level shot of a small group of friends sitting on the grass in a park, chatting on a sunny afternoon. They are wearing simple t-shirts and jeans.", | |
| "network_multiplier": "0.5", | |
| "seed": 42 | |
| }, | |
| { | |
| "prompt": "A medium, eye-level shot of a small group of friends sitting on the grass in a park, chatting on a sunny afternoon. They are wearing simple t-shirts and jeans.", | |
| "network_multiplier": "1.0", | |
| "seed": 42 | |
| }, | |
| { | |
| "prompt": "A medium, eye-level shot of a small group of friends sitting on the grass in a park, chatting on a sunny afternoon. They are wearing simple t-shirts and jeans.", | |
| "network_multiplier": "2.0", | |
| "seed": 42 | |
| } | |
| ], | |
| "neg": "", | |
| "seed": 42, | |
| "walk_seed": true, | |
| "guidance_scale": 4, | |
| "sample_steps": 25, | |
| "num_frames": 1, | |
| "fps": 1 | |
| }, | |
| "slider": { | |
| "guidance_strength": 3, | |
| "anchor_strength": 1, | |
| "positive_prompt": "A photo of a vibrant hippie festival scene, with a joyful crowd, psychedelic tie-dye patterns, and live music.", | |
| "negative_prompt": "A photo of a mundane modern scene, with organized space, plain solid colors, and a quiet atmosphere.", | |
| "target_class": "scene", | |
| "anchor_class": "" | |
| } | |
| } | |
| Using SQLite database at /app/ai-toolkit/aitk_db.db | |
| Job ID: "cb135dd2-679b-4b46-9643-6a98add9bca2" | |
| ############################################# | |
| # Running job: The_Cosmic_Drift_Slider | |
| ############################################# | |
| Running 1 process | |
| Loading Qwen Image model | |
| Loading transformer | |
| Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s] Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s] Loading checkpoint shards: 44%|####4 | 4/9 [00:00<00:00, 37.64it/s] Loading checkpoint shards: 44%|####4 | 4/9 [00:00<00:00, 37.64it/s] Loading checkpoint shards: 89%|########8 | 8/9 [00:00<00:00, 38.42it/s] Loading checkpoint shards: 89%|########8 | 8/9 [00:00<00:00, 38.42it/s] Loading checkpoint shards: 100%|##########| 9/9 [00:00<00:00, 40.76it/s] Loading checkpoint shards: 100%|##########| 9/9 [00:00<00:00, 40.76it/s] | |
| Quantizing Transformer | |
| Grabbing lora from the hub: ostris/accuracy_recovery_adapters/qwen_image_torchao_uint3.safetensors | |
| create LoRA network. base dim (rank): 16, alpha: 16 | |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None | |
| create LoRA for Text Encoder: 0 modules. | |
| create LoRA for U-Net: 846 modules. | |
| enable LoRA for U-Net | |
| Missing keys: [] | |
| Attaching quantization: 0%| | 0/846 [00:00<?, ?it/s] Attaching quantization: 0%| | 0/846 [00:00<?, ?it/s] Attaching quantization: 0%| | 4/846 [00:00<00:37, 22.51it/s] Attaching quantization: 0%| | 4/846 [00:00<00:37, 22.51it/s] Attaching quantization: 1%| | 7/846 [00:00<01:18, 10.73it/s] Attaching quantization: 1%| | 7/846 [00:00<01:18, 10.73it/s] Attaching quantization: 1%|1 | 10/846 [00:00<00:57, 14.42it/s] Attaching quantization: 1%|1 | 10/846 [00:00<00:57, 14.42it/s] Attaching quantization: 2%|1 | 14/846 [00:01<01:07, 12.27it/s] Attaching quantization: 2%|1 | 14/846 [00:01<01:07, 12.27it/s] Attaching quantization: 2%|1 | 16/846 [00:01<01:53, 7.33it/s] Attaching quantization: 2%|1 | 16/846 [00:01<01:53, 7.33it/s] Attaching quantization: 2%|2 | 18/846 [00:02<02:24, 5.72it/s] Attaching quantization: 2%|2 | 18/846 [00:02<02:24, 5.72it/s] Attaching quantization: 2%|2 | 19/846 [00:02<02:42, 5.09it/s] Attaching quantization: 2%|2 | 19/846 [00:02<02:42, 5.09it/s] Attaching quantization: 3%|2 | 23/846 [00:02<01:35, 8.59it/s] Attaching quantization: 3%|2 | 23/846 [00:02<01:35, 8.59it/s] Attaching quantization: 3%|3 | 26/846 [00:02<01:19, 10.31it/s] Attaching quantization: 3%|3 | 26/846 [00:02<01:19, 10.31it/s] Attaching quantization: 3%|3 | 28/846 [00:03<01:30, 8.99it/s] Attaching quantization: 3%|3 | 28/846 [00:03<01:30, 8.99it/s] Attaching quantization: 4%|3 | 30/846 [00:03<02:11, 6.22it/s] Attaching quantization: 4%|3 | 30/846 [00:03<02:11, 6.22it/s] Attaching quantization: 4%|3 | 32/846 [00:04<02:31, 5.39it/s] Attaching quantization: 4%|3 | 32/846 [00:04<02:31, 5.39it/s] Attaching quantization: 4%|3 | 33/846 [00:04<02:56, 4.62it/s] Attaching quantization: 4%|3 | 33/846 [00:04<02:56, 4.62it/s] Attaching quantization: 4%|4 | 36/846 [00:04<01:55, 7.02it/s] Attaching quantization: 4%|4 | 36/846 [00:04<01:55, 7.02it/s] Attaching quantization: 4%|4 | 38/846 [00:04<01:34, 8.56it/s] Attaching quantization: 4%|4 | 38/846 [00:04<01:34, 8.56it/s] Attaching quantization: 5%|4 | 41/846 [00:04<01:09, 11.62it/s] Attaching quantization: 5%|4 | 41/846 [00:04<01:09, 11.62it/s] Attaching quantization: 5%|5 | 43/846 [00:05<01:53, 7.11it/s] Attaching quantization: 5%|5 | 43/846 [00:05<01:53, 7.11it/s] Attaching quantization: 5%|5 | 45/846 [00:06<02:28, 5.40it/s] Attaching quantization: 5%|5 | 45/846 [00:06<02:28, 5.40it/s] Attaching quantization: 6%|5 | 47/846 [00:06<02:54, 4.58it/s] Attaching quantization: 6%|5 | 47/846 [00:06<02:54, 4.58it/s] Attaching quantization: 6%|5 | 50/846 [00:06<01:58, 6.70it/s] Attaching quantization: 6%|5 | 50/846 [00:06<01:58, 6.70it/s] Attaching quantization: 6%|6 | 52/846 [00:06<01:38, 8.09it/s] Attaching quantization: 6%|6 | 52/846 [00:06<01:38, 8.09it/s] Attaching quantization: 7%|6 | 55/846 [00:07<01:19, 9.89it/s] Attaching quantization: 7%|6 | 55/846 [00:07<01:19, 9.89it/s] Attaching quantization: 7%|6 | 57/846 [00:07<01:50, 7.12it/s] Attaching quantization: 7%|6 | 57/846 [00:07<01:50, 7.12it/s] Attaching quantization: 7%|6 | 59/846 [00:08<02:24, 5.46it/s] Attaching quantization: 7%|6 | 59/846 [00:08<02:24, 5.46it/s] Attaching quantization: 7%|7 | 60/846 [00:08<02:37, 4.98it/s] Attaching quantization: 7%|7 | 60/846 [00:08<02:37, 4.98it/s] Attaching quantization: 7%|7 | 61/846 [00:08<02:54, 4.49it/s] Attaching quantization: 7%|7 | 61/846 [00:08<02:54, 4.49it/s] Attaching quantization: 8%|7 | 65/846 [00:09<01:41, 7.68it/s] Attaching quantization: 8%|7 | 65/846 [00:09<01:41, 7.68it/s] Attaching quantization: 8%|8 | 68/846 [00:09<01:14, 10.44it/s] Attaching quantization: 8%|8 | 68/846 [00:09<01:14, 10.44it/s] Attaching quantization: 8%|8 | 70/846 [00:09<01:25, 9.08it/s] Attaching quantization: 8%|8 | 70/846 [00:09<01:25, 9.08it/s] Attaching quantization: 9%|8 | 72/846 [00:10<02:04, 6.23it/s] Attaching quantization: 9%|8 | 72/846 [00:10<02:04, 6.23it/s] Attaching quantization: 9%|8 | 74/846 [00:10<02:24, 5.35it/s] Attaching quantization: 9%|8 | 74/846 [00:10<02:24, 5.35it/s] Attaching quantization: 9%|8 | 75/846 [00:10<02:40, 4.81it/s] Attaching quantization: 9%|8 | 75/846 [00:10<02:40, 4.81it/s] Attaching quantization: 9%|9 | 78/846 [00:11<01:52, 6.85it/s] Attaching quantization: 9%|9 | 78/846 [00:11<01:52, 6.85it/s] Attaching quantization: 10%|9 | 82/846 [00:11<01:11, 10.73it/s] Attaching quantization: 10%|9 | 82/846 [00:11<01:11, 10.73it/s] Attaching quantization: 10%|9 | 84/846 [00:11<01:21, 9.35it/s] Attaching quantization: 10%|9 | 84/846 [00:11<01:21, 9.35it/s] Attaching quantization: 10%|# | 86/846 [00:12<02:00, 6.33it/s] Attaching quantization: 10%|# | 86/846 [00:12<02:00, 6.33it/s] Attaching quantization: 10%|# | 88/846 [00:12<02:26, 5.17it/s] Attaching quantization: 10%|# | 88/846 [00:12<02:26, 5.17it/s] Attaching quantization: 11%|# | 89/846 [00:13<02:41, 4.69it/s] Attaching quantization: 11%|# | 89/846 [00:13<02:41, 4.69it/s] Attaching quantization: 11%|# | 93/846 [00:13<01:32, 8.10it/s] Attaching quantization: 11%|# | 93/846 [00:13<01:32, 8.10it/s] Attaching quantization: 11%|#1 | 97/846 [00:13<01:08, 11.00it/s] Attaching quantization: 11%|#1 | 97/846 [00:13<01:08, 11.00it/s] Attaching quantization: 12%|#1 | 99/846 [00:13<01:36, 7.77it/s] Attaching quantization: 12%|#1 | 99/846 [00:13<01:36, 7.77it/s] Attaching quantization: 12%|#1 | 101/846 [00:14<02:07, 5.84it/s] Attaching quantization: 12%|#1 | 101/846 [00:14<02:07, 5.84it/s] Attaching quantization: 12%|#2 | 103/846 [00:15<02:39, 4.65it/s] Attaching quantization: 12%|#2 | 103/846 [00:15<02:39, 4.65it/s] Attaching quantization: 13%|#2 | 106/846 [00:15<01:51, 6.63it/s] Attaching quantization: 13%|#2 | 106/846 [00:15<01:51, 6.63it/s] Attaching quantization: 13%|#2 | 108/846 [00:15<01:32, 7.97it/s] Attaching quantization: 13%|#2 | 108/846 [00:15<01:32, 7.97it/s] Attaching quantization: 13%|#3 | 111/846 [00:15<01:08, 10.74it/s] Attaching quantization: 13%|#3 | 111/846 [00:15<01:08, 10.74it/s] Attaching quantization: 13%|#3 | 114/846 [00:16<01:59, 6.14it/s] Attaching quantization: 13%|#3 | 114/846 [00:16<01:59, 6.14it/s] Attaching quantization: 14%|#3 | 116/846 [00:16<02:14, 5.41it/s] Attaching quantization: 14%|#3 | 116/846 [00:16<02:14, 5.41it/s] Attaching quantization: 14%|#3 | 118/846 [00:17<02:18, 5.24it/s] Attaching quantization: 14%|#3 | 118/846 [00:17<02:18, 5.24it/s] Attaching quantization: 14%|#4 | 121/846 [00:17<01:42, 7.06it/s] Attaching quantization: 14%|#4 | 121/846 [00:17<01:42, 7.06it/s] Attaching quantization: 15%|#4 | 125/846 [00:17<01:08, 10.47it/s] Attaching quantization: 15%|#4 | 125/846 [00:17<01:08, 10.47it/s] Attaching quantization: 15%|#5 | 127/846 [00:17<01:34, 7.62it/s] Attaching quantization: 15%|#5 | 127/846 [00:17<01:34, 7.62it/s] Attaching quantization: 15%|#5 | 129/846 [00:18<02:10, 5.49it/s] Attaching quantization: 15%|#5 | 129/846 [00:18<02:10, 5.49it/s] Attaching quantization: 15%|#5 | 131/846 [00:19<02:33, 4.67it/s] Attaching quantization: 15%|#5 | 131/846 [00:19<02:33, 4.67it/s] Attaching quantization: 16%|#6 | 136/846 [00:19<01:26, 8.20it/s] Attaching quantization: 16%|#6 | 136/846 [00:19<01:26, 8.20it/s] Attaching quantization: 17%|#6 | 140/846 [00:19<01:19, 8.86it/s] Attaching quantization: 17%|#6 | 140/846 [00:19<01:19, 8.86it/s] Attaching quantization: 17%|#6 | 142/846 [00:20<01:46, 6.59it/s] Attaching quantization: 17%|#6 | 142/846 [00:20<01:46, 6.59it/s] Attaching quantization: 17%|#7 | 144/846 [00:20<02:02, 5.73it/s] Attaching quantization: 17%|#7 | 144/846 [00:20<02:02, 5.73it/s] Attaching quantization: 17%|#7 | 146/846 [00:21<02:07, 5.51it/s] Attaching quantization: 17%|#7 | 146/846 [00:21<02:07, 5.51it/s] Attaching quantization: 18%|#7 | 150/846 [00:21<01:26, 8.00it/s] Attaching quantization: 18%|#7 | 150/846 [00:21<01:26, 8.00it/s] Attaching quantization: 18%|#8 | 154/846 [00:21<01:15, 9.20it/s] Attaching quantization: 18%|#8 | 154/846 [00:21<01:15, 9.20it/s] Attaching quantization: 18%|#8 | 156/846 [00:22<01:48, 6.38it/s] Attaching quantization: 18%|#8 | 156/846 [00:22<01:48, 6.38it/s] Attaching quantization: 19%|#8 | 157/846 [00:22<01:53, 6.06it/s] Attaching quantization: 19%|#8 | 157/846 [00:22<01:53, 6.06it/s] Attaching quantization: 19%|#8 | 158/846 [00:22<02:07, 5.40it/s] Attaching quantization: 19%|#8 | 158/846 [00:22<02:07, 5.40it/s] Attaching quantization: 19%|#8 | 159/846 [00:23<02:32, 4.51it/s] Attaching quantization: 19%|#8 | 159/846 [00:23<02:32, 4.51it/s] Attaching quantization: 19%|#9 | 162/846 [00:23<01:36, 7.10it/s] Attaching quantization: 19%|#9 | 162/846 [00:23<01:36, 7.10it/s] Attaching quantization: 19%|#9 | 164/846 [00:23<01:18, 8.73it/s] Attaching quantization: 19%|#9 | 164/846 [00:23<01:18, 8.73it/s] Attaching quantization: 20%|#9 | 168/846 [00:23<01:12, 9.30it/s] Attaching quantization: 20%|#9 | 168/846 [00:23<01:12, 9.30it/s] Attaching quantization: 20%|## | 170/846 [00:24<01:46, 6.38it/s] Attaching quantization: 20%|## | 170/846 [00:24<01:46, 6.38it/s] Attaching quantization: 20%|## | 172/846 [00:25<02:09, 5.21it/s] Attaching quantization: 20%|## | 172/846 [00:25<02:09, 5.21it/s] Attaching quantization: 20%|## | 173/846 [00:25<02:22, 4.73it/s] Attaching quantization: 20%|## | 173/846 [00:25<02:22, 4.73it/s] Attaching quantization: 21%|## | 177/846 [00:25<01:27, 7.63it/s] Attaching quantization: 21%|## | 177/846 [00:25<01:27, 7.63it/s] Attaching quantization: 21%|##1 | 180/846 [00:25<01:05, 10.15it/s] Attaching quantization: 21%|##1 | 180/846 [00:25<01:05, 10.15it/s] Attaching quantization: 22%|##1 | 182/846 [00:26<01:14, 8.92it/s] Attaching quantization: 22%|##1 | 182/846 [00:26<01:14, 8.92it/s] Attaching quantization: 22%|##1 | 184/846 [00:26<01:47, 6.17it/s] Attaching quantization: 22%|##1 | 184/846 [00:26<01:47, 6.17it/s] Attaching quantization: 22%|##1 | 186/846 [00:27<02:09, 5.08it/s] Attaching quantization: 22%|##1 | 186/846 [00:27<02:09, 5.08it/s] Attaching quantization: 22%|##2 | 187/846 [00:27<02:23, 4.60it/s] Attaching quantization: 22%|##2 | 187/846 [00:27<02:23, 4.60it/s] Attaching quantization: 23%|##2 | 191/846 [00:27<01:27, 7.49it/s] Attaching quantization: 23%|##2 | 191/846 [00:27<01:27, 7.49it/s] Attaching quantization: 23%|##2 | 194/846 [00:27<01:04, 10.05it/s] Attaching quantization: 23%|##2 | 194/846 [00:27<01:04, 10.05it/s] Attaching quantization: 23%|##3 | 196/846 [00:28<01:12, 8.93it/s] Attaching quantization: 23%|##3 | 196/846 [00:28<01:12, 8.93it/s] Attaching quantization: 23%|##3 | 198/846 [00:28<01:45, 6.16it/s] Attaching quantization: 23%|##3 | 198/846 [00:28<01:45, 6.16it/s] Attaching quantization: 24%|##3 | 200/846 [00:29<02:00, 5.37it/s] Attaching quantization: 24%|##3 | 200/846 [00:29<02:00, 5.37it/s] Attaching quantization: 24%|##3 | 201/846 [00:29<02:21, 4.54it/s] Attaching quantization: 24%|##3 | 201/846 [00:29<02:21, 4.54it/s] Attaching quantization: 24%|##4 | 205/846 [00:29<01:21, 7.85it/s] Attaching quantization: 24%|##4 | 205/846 [00:29<01:21, 7.85it/s] Attaching quantization: 25%|##4 | 209/846 [00:29<00:54, 11.59it/s] Attaching quantization: 25%|##4 | 209/846 [00:29<00:54, 11.59it/s] Attaching quantization: 25%|##5 | 212/846 [00:30<01:35, 6.63it/s] Attaching quantization: 25%|##5 | 212/846 [00:30<01:35, 6.63it/s] Attaching quantization: 25%|##5 | 214/846 [00:31<01:55, 5.49it/s] Attaching quantization: 25%|##5 | 214/846 [00:31<01:55, 5.49it/s] Attaching quantization: 26%|##5 | 216/846 [00:31<01:52, 5.62it/s] Attaching quantization: 26%|##5 | 216/846 [00:31<01:52, 5.62it/s] Attaching quantization: 26%|##6 | 220/846 [00:31<01:16, 8.21it/s] Attaching quantization: 26%|##6 | 220/846 [00:31<01:16, 8.21it/s] Attaching quantization: 26%|##6 | 224/846 [00:32<01:05, 9.43it/s] Attaching quantization: 26%|##6 | 224/846 [00:32<01:05, 9.43it/s] Attaching quantization: 27%|##6 | 226/846 [00:32<01:31, 6.75it/s] Attaching quantization: 27%|##6 | 226/846 [00:32<01:31, 6.75it/s] Attaching quantization: 27%|##6 | 228/846 [00:33<01:51, 5.54it/s] Attaching quantization: 27%|##6 | 228/846 [00:33<01:51, 5.54it/s] Attaching quantization: 27%|##7 | 229/846 [00:33<02:03, 5.01it/s] Attaching quantization: 27%|##7 | 229/846 [00:33<02:03, 5.01it/s] Attaching quantization: 28%|##7 | 233/846 [00:33<01:14, 8.18it/s] Attaching quantization: 28%|##7 | 233/846 [00:33<01:14, 8.18it/s] Attaching quantization: 28%|##8 | 237/846 [00:33<00:55, 10.89it/s] Attaching quantization: 28%|##8 | 237/846 [00:33<00:55, 10.89it/s] Attaching quantization: 28%|##8 | 239/846 [00:34<01:17, 7.87it/s] Attaching quantization: 28%|##8 | 239/846 [00:34<01:17, 7.87it/s] Attaching quantization: 28%|##8 | 241/846 [00:35<01:41, 5.95it/s] Attaching quantization: 28%|##8 | 241/846 [00:35<01:41, 5.95it/s] Attaching quantization: 29%|##8 | 243/846 [00:35<02:01, 4.94it/s] Attaching quantization: 29%|##8 | 243/846 [00:35<02:01, 4.94it/s] Attaching quantization: 29%|##9 | 247/846 [00:35<01:16, 7.80it/s] Attaching quantization: 29%|##9 | 247/846 [00:35<01:16, 7.80it/s] Attaching quantization: 30%|##9 | 251/846 [00:35<00:57, 10.33it/s] Attaching quantization: 30%|##9 | 251/846 [00:35<00:57, 10.33it/s] Attaching quantization: 30%|##9 | 253/846 [00:36<01:17, 7.65it/s] Attaching quantization: 30%|##9 | 253/846 [00:36<01:17, 7.65it/s] Attaching quantization: 30%|### | 255/846 [00:37<01:48, 5.47it/s] Attaching quantization: 30%|### | 255/846 [00:37<01:48, 5.47it/s] Attaching quantization: 30%|### | 257/846 [00:37<02:19, 4.23it/s] Attaching quantization: 30%|### | 257/846 [00:37<02:19, 4.23it/s] Attaching quantization: 31%|### | 262/846 [00:38<01:22, 7.04it/s] Attaching quantization: 31%|### | 262/846 [00:38<01:22, 7.04it/s] Attaching quantization: 31%|###1 | 264/846 [00:38<01:18, 7.43it/s] Attaching quantization: 31%|###1 | 264/846 [00:38<01:18, 7.43it/s] Attaching quantization: 31%|###1 | 266/846 [00:38<01:27, 6.60it/s] Attaching quantization: 31%|###1 | 266/846 [00:38<01:27, 6.60it/s] Attaching quantization: 32%|###1 | 268/846 [00:39<02:02, 4.71it/s] Attaching quantization: 32%|###1 | 268/846 [00:39<02:02, 4.71it/s] Attaching quantization: 32%|###1 | 269/846 [00:39<02:08, 4.50it/s] Attaching quantization: 32%|###1 | 269/846 [00:39<02:08, 4.50it/s] Attaching quantization: 32%|###1 | 270/846 [00:40<02:08, 4.47it/s] Attaching quantization: 32%|###1 | 270/846 [00:40<02:08, 4.47it/s] Attaching quantization: 32%|###2 | 271/846 [00:40<02:40, 3.59it/s] Attaching quantization: 32%|###2 | 271/846 [00:40<02:40, 3.59it/s] Attaching quantization: 33%|###2 | 276/846 [00:40<01:16, 7.43it/s] Attaching quantization: 33%|###2 | 276/846 [00:40<01:16, 7.43it/s] Attaching quantization: 33%|###3 | 280/846 [00:41<01:09, 8.12it/s] Attaching quantization: 33%|###3 | 280/846 [00:41<01:09, 8.12it/s] Attaching quantization: 33%|###3 | 282/846 [00:42<01:44, 5.39it/s] Attaching quantization: 33%|###3 | 282/846 [00:42<01:44, 5.39it/s] Attaching quantization: 33%|###3 | 283/846 [00:42<01:52, 5.00it/s] Attaching quantization: 33%|###3 | 283/846 [00:42<01:52, 5.00it/s] Attaching quantization: 34%|###3 | 284/846 [00:42<02:01, 4.62it/s] Attaching quantization: 34%|###3 | 284/846 [00:42<02:01, 4.62it/s] Attaching quantization: 34%|###3 | 285/846 [00:43<02:34, 3.63it/s] Attaching quantization: 34%|###3 | 285/846 [00:43<02:34, 3.63it/s] Attaching quantization: 34%|###4 | 289/846 [00:43<01:24, 6.59it/s] Attaching quantization: 34%|###4 | 289/846 [00:43<01:24, 6.59it/s] Attaching quantization: 35%|###4 | 293/846 [00:43<00:54, 10.19it/s] Attaching quantization: 35%|###4 | 293/846 [00:43<00:54, 10.19it/s] Attaching quantization: 35%|###4 | 295/846 [00:44<01:33, 5.86it/s] Attaching quantization: 35%|###4 | 295/846 [00:44<01:33, 5.86it/s] Attaching quantization: 35%|###5 | 297/846 [00:44<01:51, 4.91it/s] Attaching quantization: 35%|###5 | 297/846 [00:44<01:51, 4.91it/s] Attaching quantization: 35%|###5 | 299/846 [00:45<02:27, 3.71it/s] Attaching quantization: 35%|###5 | 299/846 [00:45<02:27, 3.71it/s] Attaching quantization: 36%|###5 | 303/846 [00:45<01:33, 5.82it/s] Attaching quantization: 36%|###5 | 303/846 [00:45<01:33, 5.82it/s] Attaching quantization: 36%|###6 | 307/846 [00:45<01:03, 8.54it/s] Attaching quantization: 36%|###6 | 307/846 [00:45<01:03, 8.54it/s] Attaching quantization: 37%|###6 | 309/846 [00:46<01:29, 6.00it/s] Attaching quantization: 37%|###6 | 309/846 [00:46<01:29, 6.00it/s] Attaching quantization: 37%|###6 | 311/846 [00:47<01:53, 4.73it/s] Attaching quantization: 37%|###6 | 311/846 [00:47<01:53, 4.73it/s] Attaching quantization: 37%|###6 | 313/846 [00:48<02:17, 3.88it/s] Attaching quantization: 37%|###6 | 313/846 [00:48<02:17, 3.88it/s] Attaching quantization: 37%|###7 | 317/846 [00:48<01:25, 6.18it/s] Attaching quantization: 37%|###7 | 317/846 [00:48<01:25, 6.18it/s] Attaching quantization: 38%|###7 | 319/846 [00:48<01:17, 6.78it/s] Attaching quantization: 38%|###7 | 319/846 [00:48<01:17, 6.78it/s] Attaching quantization: 38%|###8 | 322/846 [00:48<01:20, 6.53it/s] Attaching quantization: 38%|###8 | 322/846 [00:48<01:20, 6.53it/s] Attaching quantization: 38%|###8 | 324/846 [00:49<01:55, 4.51it/s] Attaching quantization: 38%|###8 | 324/846 [00:49<01:55, 4.51it/s] Attaching quantization: 38%|###8 | 325/846 [00:50<01:55, 4.53it/s] Attaching quantization: 38%|###8 | 325/846 [00:50<01:55, 4.53it/s] Attaching quantization: 39%|###8 | 326/846 [00:50<02:01, 4.27it/s] Attaching quantization: 39%|###8 | 326/846 [00:50<02:01, 4.27it/s] Attaching quantization: 39%|###8 | 327/846 [00:50<02:30, 3.45it/s] Attaching quantization: 39%|###8 | 327/846 [00:50<02:30, 3.45it/s] Attaching quantization: 39%|###9 | 331/846 [00:51<01:21, 6.32it/s] Attaching quantization: 39%|###9 | 331/846 [00:51<01:21, 6.32it/s] Attaching quantization: 39%|###9 | 333/846 [00:51<01:07, 7.64it/s] Attaching quantization: 39%|###9 | 333/846 [00:51<01:07, 7.64it/s] Attaching quantization: 40%|###9 | 336/846 [00:51<01:13, 6.98it/s] Attaching quantization: 40%|###9 | 336/846 [00:51<01:13, 6.98it/s] Attaching quantization: 40%|###9 | 338/846 [00:52<01:41, 4.99it/s] Attaching quantization: 40%|###9 | 338/846 [00:52<01:41, 4.99it/s] Attaching quantization: 40%|#### | 339/846 [00:52<01:47, 4.70it/s] Attaching quantization: 40%|#### | 339/846 [00:52<01:47, 4.70it/s] Attaching quantization: 40%|#### | 340/846 [00:52<01:56, 4.36it/s] Attaching quantization: 40%|#### | 340/846 [00:52<01:56, 4.36it/s] Attaching quantization: 40%|#### | 341/846 [00:53<02:12, 3.82it/s] Attaching quantization: 40%|#### | 341/846 [00:53<02:12, 3.82it/s] Attaching quantization: 41%|#### | 345/846 [00:53<01:06, 7.51it/s] Attaching quantization: 41%|#### | 345/846 [00:53<01:06, 7.51it/s] Attaching quantization: 41%|####1 | 348/846 [00:53<00:48, 10.26it/s] Attaching quantization: 41%|####1 | 348/846 [00:53<00:48, 10.26it/s] Attaching quantization: 41%|####1 | 350/846 [00:53<00:56, 8.83it/s] Attaching quantization: 41%|####1 | 350/846 [00:53<00:56, 8.83it/s] Attaching quantization: 42%|####1 | 352/846 [00:54<01:26, 5.73it/s] Attaching quantization: 42%|####1 | 352/846 [00:54<01:26, 5.73it/s] Attaching quantization: 42%|####1 | 354/846 [00:55<01:37, 5.07it/s] Attaching quantization: 42%|####1 | 354/846 [00:55<01:37, 5.07it/s] Attaching quantization: 42%|####1 | 355/846 [00:55<01:53, 4.33it/s] Attaching quantization: 42%|####1 | 355/846 [00:55<01:53, 4.33it/s] Attaching quantization: 42%|####2 | 359/846 [00:55<01:03, 7.63it/s] Attaching quantization: 42%|####2 | 359/846 [00:55<01:03, 7.63it/s] Attaching quantization: 43%|####2 | 363/846 [00:55<00:46, 10.48it/s] Attaching quantization: 43%|####2 | 363/846 [00:55<00:46, 10.48it/s] Attaching quantization: 43%|####3 | 365/846 [00:56<01:08, 7.01it/s] Attaching quantization: 43%|####3 | 365/846 [00:56<01:08, 7.01it/s] Attaching quantization: 43%|####3 | 367/846 [00:57<01:27, 5.45it/s] Attaching quantization: 43%|####3 | 367/846 [00:57<01:27, 5.45it/s] Attaching quantization: 44%|####3 | 369/846 [00:57<01:47, 4.43it/s] Attaching quantization: 44%|####3 | 369/846 [00:57<01:47, 4.43it/s] Attaching quantization: 44%|####4 | 373/846 [00:57<01:06, 7.10it/s] Attaching quantization: 44%|####4 | 373/846 [00:57<01:06, 7.10it/s] Attaching quantization: 44%|####4 | 376/846 [00:57<00:54, 8.69it/s] Attaching quantization: 44%|####4 | 376/846 [00:57<00:54, 8.69it/s] Attaching quantization: 45%|####4 | 378/846 [00:58<00:58, 8.02it/s] Attaching quantization: 45%|####4 | 378/846 [00:58<00:58, 8.02it/s] Attaching quantization: 45%|####4 | 380/846 [00:58<01:24, 5.52it/s] Attaching quantization: 45%|####4 | 380/846 [00:58<01:24, 5.52it/s] Attaching quantization: 45%|####5 | 382/846 [00:59<01:32, 5.00it/s] Attaching quantization: 45%|####5 | 382/846 [00:59<01:32, 5.00it/s] Attaching quantization: 45%|####5 | 383/846 [00:59<01:47, 4.30it/s] Attaching quantization: 45%|####5 | 383/846 [00:59<01:47, 4.30it/s] Attaching quantization: 46%|####5 | 387/846 [01:00<01:05, 7.03it/s] Attaching quantization: 46%|####5 | 387/846 [01:00<01:05, 7.03it/s] Attaching quantization: 46%|####6 | 390/846 [01:00<00:48, 9.43it/s] Attaching quantization: 46%|####6 | 390/846 [01:00<00:48, 9.43it/s] Attaching quantization: 46%|####6 | 392/846 [01:00<00:53, 8.42it/s] Attaching quantization: 46%|####6 | 392/846 [01:00<00:53, 8.42it/s] Attaching quantization: 47%|####6 | 394/846 [01:01<01:20, 5.59it/s] Attaching quantization: 47%|####6 | 394/846 [01:01<01:20, 5.59it/s] Attaching quantization: 47%|####6 | 396/846 [01:01<01:34, 4.76it/s] Attaching quantization: 47%|####6 | 396/846 [01:01<01:34, 4.76it/s] Attaching quantization: 47%|####6 | 397/846 [01:02<01:48, 4.14it/s] Attaching quantization: 47%|####6 | 397/846 [01:02<01:48, 4.14it/s] Attaching quantization: 47%|####7 | 401/846 [01:02<01:01, 7.26it/s] Attaching quantization: 47%|####7 | 401/846 [01:02<01:01, 7.26it/s] Attaching quantization: 48%|####7 | 404/846 [01:02<00:49, 9.01it/s] Attaching quantization: 48%|####7 | 404/846 [01:02<00:49, 9.01it/s] Attaching quantization: 48%|####7 | 406/846 [01:02<00:53, 8.26it/s] Attaching quantization: 48%|####7 | 406/846 [01:02<00:53, 8.26it/s] Attaching quantization: 48%|####8 | 408/846 [01:03<01:13, 5.93it/s] Attaching quantization: 48%|####8 | 408/846 [01:03<01:13, 5.93it/s] Attaching quantization: 48%|####8 | 410/846 [01:03<01:17, 5.64it/s] Attaching quantization: 48%|####8 | 410/846 [01:03<01:17, 5.64it/s] Attaching quantization: 49%|####8 | 411/846 [01:04<01:33, 4.65it/s] Attaching quantization: 49%|####8 | 411/846 [01:04<01:33, 4.65it/s] Attaching quantization: 49%|####9 | 415/846 [01:04<00:53, 8.06it/s] Attaching quantization: 49%|####9 | 415/846 [01:04<00:53, 8.06it/s] Attaching quantization: 49%|####9 | 418/846 [01:04<00:50, 8.46it/s] Attaching quantization: 49%|####9 | 418/846 [01:04<00:50, 8.46it/s] Attaching quantization: 50%|####9 | 420/846 [01:05<01:01, 6.92it/s] Attaching quantization: 50%|####9 | 420/846 [01:05<01:01, 6.92it/s] Attaching quantization: 50%|####9 | 422/846 [01:05<01:24, 4.99it/s] Attaching quantization: 50%|####9 | 422/846 [01:05<01:24, 4.99it/s] Attaching quantization: 50%|##### | 423/846 [01:06<01:30, 4.68it/s] Attaching quantization: 50%|##### | 423/846 [01:06<01:30, 4.68it/s] Attaching quantization: 50%|##### | 424/846 [01:06<01:30, 4.64it/s] Attaching quantization: 50%|##### | 424/846 [01:06<01:30, 4.64it/s] Attaching quantization: 50%|##### | 425/846 [01:06<01:55, 3.65it/s] Attaching quantization: 50%|##### | 425/846 [01:06<01:55, 3.65it/s] Attaching quantization: 51%|##### | 430/846 [01:06<00:54, 7.64it/s] Attaching quantization: 51%|##### | 430/846 [01:06<00:54, 7.64it/s] Attaching quantization: 51%|#####1 | 432/846 [01:07<00:46, 8.88it/s] Attaching quantization: 51%|#####1 | 432/846 [01:07<00:46, 8.88it/s] Attaching quantization: 51%|#####1 | 434/846 [01:07<01:01, 6.74it/s] Attaching quantization: 51%|#####1 | 434/846 [01:07<01:01, 6.74it/s] Attaching quantization: 52%|#####1 | 436/846 [01:08<01:23, 4.88it/s] Attaching quantization: 52%|#####1 | 436/846 [01:08<01:23, 4.88it/s] Attaching quantization: 52%|#####1 | 437/846 [01:08<01:24, 4.83it/s] Attaching quantization: 52%|#####1 | 437/846 [01:08<01:24, 4.83it/s] Attaching quantization: 52%|#####1 | 438/846 [01:08<01:44, 3.89it/s] Attaching quantization: 52%|#####1 | 438/846 [01:08<01:44, 3.89it/s] Attaching quantization: 52%|#####1 | 439/846 [01:09<01:50, 3.69it/s] Attaching quantization: 52%|#####1 | 439/846 [01:09<01:50, 3.69it/s] Attaching quantization: 52%|#####2 | 444/846 [01:09<00:51, 7.79it/s] Attaching quantization: 52%|#####2 | 444/846 [01:09<00:51, 7.79it/s] Attaching quantization: 53%|#####2 | 446/846 [01:09<00:43, 9.22it/s] Attaching quantization: 53%|#####2 | 446/846 [01:09<00:43, 9.22it/s] Attaching quantization: 53%|#####2 | 448/846 [01:09<00:53, 7.42it/s] Attaching quantization: 53%|#####2 | 448/846 [01:09<00:53, 7.42it/s] Attaching quantization: 53%|#####3 | 450/846 [01:10<01:28, 4.49it/s] Attaching quantization: 53%|#####3 | 450/846 [01:10<01:28, 4.49it/s] Attaching quantization: 53%|#####3 | 451/846 [01:11<01:32, 4.28it/s] Attaching quantization: 53%|#####3 | 451/846 [01:11<01:32, 4.28it/s] Attaching quantization: 53%|#####3 | 452/846 [01:11<01:31, 4.32it/s] Attaching quantization: 53%|#####3 | 452/846 [01:11<01:31, 4.32it/s] Attaching quantization: 54%|#####3 | 453/846 [01:11<01:52, 3.48it/s] Attaching quantization: 54%|#####3 | 453/846 [01:11<01:52, 3.48it/s] Attaching quantization: 54%|#####4 | 457/846 [01:12<00:55, 6.98it/s] Attaching quantization: 54%|#####4 | 457/846 [01:12<00:55, 6.98it/s] Attaching quantization: 54%|#####4 | 460/846 [01:12<00:43, 8.89it/s] Attaching quantization: 54%|#####4 | 460/846 [01:12<00:43, 8.89it/s] Attaching quantization: 55%|#####4 | 462/846 [01:12<00:48, 7.98it/s] Attaching quantization: 55%|#####4 | 462/846 [01:12<00:48, 7.98it/s] Attaching quantization: 55%|#####4 | 464/846 [01:13<01:11, 5.36it/s] Attaching quantization: 55%|#####4 | 464/846 [01:13<01:11, 5.36it/s] Attaching quantization: 55%|#####4 | 465/846 [01:13<01:17, 4.93it/s] Attaching quantization: 55%|#####4 | 465/846 [01:13<01:17, 4.93it/s] Attaching quantization: 55%|#####5 | 466/846 [01:13<01:24, 4.52it/s] Attaching quantization: 55%|#####5 | 466/846 [01:13<01:24, 4.52it/s] Attaching quantization: 55%|#####5 | 467/846 [01:14<01:39, 3.80it/s] Attaching quantization: 55%|#####5 | 467/846 [01:14<01:39, 3.80it/s] Attaching quantization: 56%|#####5 | 471/846 [01:14<00:53, 7.07it/s] Attaching quantization: 56%|#####5 | 471/846 [01:14<00:53, 7.07it/s] Attaching quantization: 56%|#####6 | 475/846 [01:14<00:34, 10.84it/s] Attaching quantization: 56%|#####6 | 475/846 [01:14<00:34, 10.84it/s] Attaching quantization: 56%|#####6 | 477/846 [01:15<00:56, 6.55it/s] Attaching quantization: 56%|#####6 | 477/846 [01:15<00:56, 6.55it/s] Attaching quantization: 57%|#####6 | 479/846 [01:15<01:10, 5.18it/s] Attaching quantization: 57%|#####6 | 479/846 [01:15<01:10, 5.18it/s] Attaching quantization: 57%|#####6 | 481/846 [01:16<01:25, 4.25it/s] Attaching quantization: 57%|#####6 | 481/846 [01:16<01:25, 4.25it/s] Attaching quantization: 57%|#####7 | 485/846 [01:16<00:54, 6.61it/s] Attaching quantization: 57%|#####7 | 485/846 [01:16<00:54, 6.61it/s] Attaching quantization: 58%|#####7 | 488/846 [01:16<00:41, 8.71it/s] Attaching quantization: 58%|#####7 | 488/846 [01:16<00:41, 8.71it/s] Attaching quantization: 58%|#####7 | 490/846 [01:17<00:47, 7.46it/s] Attaching quantization: 58%|#####7 | 490/846 [01:17<00:47, 7.46it/s] Attaching quantization: 58%|#####8 | 492/846 [01:17<01:07, 5.28it/s] Attaching quantization: 58%|#####8 | 492/846 [01:17<01:07, 5.28it/s] Attaching quantization: 58%|#####8 | 494/846 [01:18<01:16, 4.60it/s] Attaching quantization: 58%|#####8 | 494/846 [01:18<01:16, 4.60it/s] Attaching quantization: 59%|#####8 | 495/846 [01:18<01:27, 4.03it/s] Attaching quantization: 59%|#####8 | 495/846 [01:18<01:27, 4.03it/s] Attaching quantization: 59%|#####8 | 499/846 [01:19<00:49, 7.01it/s] Attaching quantization: 59%|#####8 | 499/846 [01:19<00:49, 7.01it/s] Attaching quantization: 59%|#####9 | 502/846 [01:19<00:36, 9.45it/s] Attaching quantization: 59%|#####9 | 502/846 [01:19<00:36, 9.45it/s] Attaching quantization: 60%|#####9 | 505/846 [01:19<00:50, 6.78it/s] Attaching quantization: 60%|#####9 | 505/846 [01:19<00:50, 6.78it/s] Attaching quantization: 60%|#####9 | 507/846 [01:20<01:05, 5.18it/s] Attaching quantization: 60%|#####9 | 507/846 [01:20<01:05, 5.18it/s] Attaching quantization: 60%|###### | 509/846 [01:21<01:18, 4.28it/s] Attaching quantization: 60%|###### | 509/846 [01:21<01:18, 4.28it/s] Attaching quantization: 61%|###### | 514/846 [01:21<00:44, 7.51it/s] Attaching quantization: 61%|###### | 514/846 [01:21<00:44, 7.51it/s] Attaching quantization: 61%|######1 | 518/846 [01:21<00:39, 8.24it/s] Attaching quantization: 61%|######1 | 518/846 [01:21<00:39, 8.24it/s] Attaching quantization: 61%|######1 | 520/846 [01:22<00:54, 5.96it/s] Attaching quantization: 61%|######1 | 520/846 [01:22<00:54, 5.96it/s] Attaching quantization: 62%|######1 | 522/846 [01:22<01:03, 5.12it/s] Attaching quantization: 62%|######1 | 522/846 [01:22<01:03, 5.12it/s] Attaching quantization: 62%|######1 | 524/846 [01:23<01:04, 5.01it/s] Attaching quantization: 62%|######1 | 524/846 [01:23<01:04, 5.01it/s] Attaching quantization: 62%|######2 | 528/846 [01:23<00:42, 7.43it/s] Attaching quantization: 62%|######2 | 528/846 [01:23<00:42, 7.43it/s] Attaching quantization: 63%|######2 | 532/846 [01:23<00:38, 8.09it/s] Attaching quantization: 63%|######2 | 532/846 [01:23<00:38, 8.09it/s] Attaching quantization: 63%|######3 | 534/846 [01:24<00:53, 5.83it/s] Attaching quantization: 63%|######3 | 534/846 [01:24<00:53, 5.83it/s] Attaching quantization: 63%|######3 | 535/846 [01:24<00:58, 5.36it/s] Attaching quantization: 63%|######3 | 535/846 [01:24<00:58, 5.36it/s] Attaching quantization: 63%|######3 | 536/846 [01:25<01:03, 4.90it/s] Attaching quantization: 63%|######3 | 536/846 [01:25<01:03, 4.90it/s] Attaching quantization: 63%|######3 | 537/846 [01:25<01:14, 4.17it/s] Attaching quantization: 63%|######3 | 537/846 [01:25<01:14, 4.17it/s] Attaching quantization: 64%|######4 | 542/846 [01:25<00:35, 8.58it/s] Attaching quantization: 64%|######4 | 542/846 [01:25<00:35, 8.58it/s] Attaching quantization: 64%|######4 | 545/846 [01:25<00:29, 10.26it/s] Attaching quantization: 64%|######4 | 545/846 [01:25<00:29, 10.26it/s] Attaching quantization: 65%|######4 | 547/846 [01:26<00:44, 6.80it/s] Attaching quantization: 65%|######4 | 547/846 [01:26<00:44, 6.80it/s] Attaching quantization: 65%|######4 | 549/846 [01:27<00:58, 5.07it/s] Attaching quantization: 65%|######4 | 549/846 [01:27<00:58, 5.07it/s] Attaching quantization: 65%|######5 | 550/846 [01:27<00:59, 4.98it/s] Attaching quantization: 65%|######5 | 550/846 [01:27<00:59, 4.98it/s] Attaching quantization: 65%|######5 | 551/846 [01:27<01:09, 4.22it/s] Attaching quantization: 65%|######5 | 551/846 [01:27<01:09, 4.22it/s] Attaching quantization: 66%|######5 | 555/846 [01:27<00:37, 7.72it/s] Attaching quantization: 66%|######5 | 555/846 [01:27<00:37, 7.72it/s] Attaching quantization: 66%|######5 | 558/846 [01:28<00:30, 9.51it/s] Attaching quantization: 66%|######5 | 558/846 [01:28<00:30, 9.51it/s] Attaching quantization: 66%|######6 | 560/846 [01:28<00:34, 8.36it/s] Attaching quantization: 66%|######6 | 560/846 [01:28<00:34, 8.36it/s] Attaching quantization: 66%|######6 | 562/846 [01:29<00:50, 5.57it/s] Attaching quantization: 66%|######6 | 562/846 [01:29<00:50, 5.57it/s] Attaching quantization: 67%|######6 | 564/846 [01:29<00:59, 4.71it/s] Attaching quantization: 67%|######6 | 564/846 [01:29<00:59, 4.71it/s] Attaching quantization: 67%|######6 | 565/846 [01:30<01:08, 4.12it/s] Attaching quantization: 67%|######6 | 565/846 [01:30<01:08, 4.12it/s] Attaching quantization: 67%|######7 | 570/846 [01:30<00:34, 8.08it/s] Attaching quantization: 67%|######7 | 570/846 [01:30<00:34, 8.08it/s] Attaching quantization: 68%|######7 | 573/846 [01:30<00:28, 9.71it/s] Attaching quantization: 68%|######7 | 573/846 [01:30<00:28, 9.71it/s] Attaching quantization: 68%|######7 | 575/846 [01:31<00:40, 6.66it/s] Attaching quantization: 68%|######7 | 575/846 [01:31<00:40, 6.66it/s] Attaching quantization: 68%|######8 | 577/846 [01:31<00:53, 5.02it/s] Attaching quantization: 68%|######8 | 577/846 [01:31<00:53, 5.02it/s] Attaching quantization: 68%|######8 | 579/846 [01:32<01:01, 4.37it/s] Attaching quantization: 68%|######8 | 579/846 [01:32<01:01, 4.37it/s] Attaching quantization: 69%|######8 | 583/846 [01:32<00:39, 6.74it/s] Attaching quantization: 69%|######8 | 583/846 [01:32<00:39, 6.74it/s] Attaching quantization: 69%|######9 | 586/846 [01:32<00:29, 8.90it/s] Attaching quantization: 69%|######9 | 586/846 [01:32<00:29, 8.90it/s] Attaching quantization: 70%|######9 | 588/846 [01:33<00:34, 7.52it/s] Attaching quantization: 70%|######9 | 588/846 [01:33<00:34, 7.52it/s] Attaching quantization: 70%|######9 | 590/846 [01:33<00:45, 5.67it/s] Attaching quantization: 70%|######9 | 590/846 [01:33<00:45, 5.67it/s] Attaching quantization: 70%|######9 | 592/846 [01:34<00:52, 4.81it/s] Attaching quantization: 70%|######9 | 592/846 [01:34<00:52, 4.81it/s] Attaching quantization: 70%|####### | 593/846 [01:34<01:00, 4.19it/s] Attaching quantization: 70%|####### | 593/846 [01:34<01:00, 4.19it/s] Attaching quantization: 71%|####### | 597/846 [01:34<00:34, 7.26it/s] Attaching quantization: 71%|####### | 597/846 [01:34<00:34, 7.26it/s] Attaching quantization: 71%|#######1 | 601/846 [01:34<00:24, 9.97it/s] Attaching quantization: 71%|#######1 | 601/846 [01:34<00:24, 9.97it/s] Attaching quantization: 71%|#######1 | 603/846 [01:35<00:35, 6.85it/s] Attaching quantization: 71%|#######1 | 603/846 [01:35<00:35, 6.85it/s] Attaching quantization: 72%|#######1 | 605/846 [01:36<00:44, 5.43it/s] Attaching quantization: 72%|#######1 | 605/846 [01:36<00:44, 5.43it/s] Attaching quantization: 72%|#######1 | 607/846 [01:36<00:54, 4.40it/s] Attaching quantization: 72%|#######1 | 607/846 [01:36<00:54, 4.40it/s] Attaching quantization: 72%|#######2 | 612/846 [01:36<00:30, 7.78it/s] Attaching quantization: 72%|#######2 | 612/846 [01:36<00:30, 7.78it/s] Attaching quantization: 73%|#######2 | 616/846 [01:37<00:28, 7.97it/s] Attaching quantization: 73%|#######2 | 616/846 [01:37<00:28, 7.97it/s] Attaching quantization: 73%|#######3 | 618/846 [01:38<00:37, 6.16it/s] Attaching quantization: 73%|#######3 | 618/846 [01:38<00:37, 6.16it/s] Attaching quantization: 73%|#######3 | 620/846 [01:38<00:41, 5.46it/s] Attaching quantization: 73%|#######3 | 620/846 [01:38<00:41, 5.46it/s] Attaching quantization: 73%|#######3 | 621/846 [01:39<00:47, 4.70it/s] Attaching quantization: 73%|#######3 | 621/846 [01:39<00:47, 4.70it/s] Attaching quantization: 74%|#######3 | 626/846 [01:39<00:31, 7.04it/s] Attaching quantization: 74%|#######3 | 626/846 [01:39<00:31, 7.04it/s] Attaching quantization: 74%|#######4 | 630/846 [01:39<00:26, 8.29it/s] Attaching quantization: 74%|#######4 | 630/846 [01:39<00:26, 8.29it/s] Attaching quantization: 75%|#######4 | 632/846 [01:40<00:33, 6.31it/s] Attaching quantization: 75%|#######4 | 632/846 [01:40<00:33, 6.31it/s] Attaching quantization: 75%|#######4 | 633/846 [01:40<00:37, 5.71it/s] Attaching quantization: 75%|#######4 | 633/846 [01:40<00:37, 5.71it/s] Attaching quantization: 75%|#######4 | 634/846 [01:40<00:38, 5.49it/s] Attaching quantization: 75%|#######4 | 634/846 [01:40<00:38, 5.49it/s] Attaching quantization: 75%|#######5 | 635/846 [01:41<00:46, 4.56it/s] Attaching quantization: 75%|#######5 | 635/846 [01:41<00:46, 4.56it/s] Attaching quantization: 76%|#######5 | 639/846 [01:41<00:25, 8.26it/s] Attaching quantization: 76%|#######5 | 639/846 [01:41<00:25, 8.26it/s] Attaching quantization: 76%|#######6 | 643/846 [01:41<00:16, 12.31it/s] Attaching quantization: 76%|#######6 | 643/846 [01:41<00:16, 12.31it/s] Attaching quantization: 76%|#######6 | 646/846 [01:42<00:31, 6.39it/s] Attaching quantization: 76%|#######6 | 646/846 [01:42<00:31, 6.39it/s] Attaching quantization: 77%|#######6 | 648/846 [01:42<00:35, 5.58it/s] Attaching quantization: 77%|#######6 | 648/846 [01:42<00:35, 5.58it/s] Attaching quantization: 77%|#######6 | 650/846 [01:43<00:35, 5.46it/s] Attaching quantization: 77%|#######6 | 650/846 [01:43<00:35, 5.46it/s] Attaching quantization: 77%|#######7 | 653/846 [01:43<00:25, 7.58it/s] Attaching quantization: 77%|#######7 | 653/846 [01:43<00:25, 7.58it/s] Attaching quantization: 78%|#######7 | 656/846 [01:43<00:19, 10.00it/s] Attaching quantization: 78%|#######7 | 656/846 [01:43<00:19, 10.00it/s] Attaching quantization: 78%|#######7 | 658/846 [01:43<00:21, 8.79it/s] Attaching quantization: 78%|#######7 | 658/846 [01:43<00:21, 8.79it/s] Attaching quantization: 78%|#######8 | 660/846 [01:44<00:32, 5.79it/s] Attaching quantization: 78%|#######8 | 660/846 [01:44<00:32, 5.79it/s] Attaching quantization: 78%|#######8 | 662/846 [01:45<00:35, 5.13it/s] Attaching quantization: 78%|#######8 | 662/846 [01:45<00:35, 5.13it/s] Attaching quantization: 78%|#######8 | 663/846 [01:45<00:41, 4.39it/s] Attaching quantization: 78%|#######8 | 663/846 [01:45<00:41, 4.39it/s] Attaching quantization: 79%|#######8 | 667/846 [01:45<00:24, 7.18it/s] Attaching quantization: 79%|#######8 | 667/846 [01:45<00:24, 7.18it/s] Attaching quantization: 79%|#######9 | 670/846 [01:45<00:18, 9.48it/s] Attaching quantization: 79%|#######9 | 670/846 [01:45<00:18, 9.48it/s] Attaching quantization: 79%|#######9 | 672/846 [01:46<00:20, 8.51it/s] Attaching quantization: 79%|#######9 | 672/846 [01:46<00:20, 8.51it/s] Attaching quantization: 80%|#######9 | 674/846 [01:46<00:30, 5.61it/s] Attaching quantization: 80%|#######9 | 674/846 [01:46<00:30, 5.61it/s] Attaching quantization: 80%|#######9 | 676/846 [01:47<00:35, 4.79it/s] Attaching quantization: 80%|#######9 | 676/846 [01:47<00:35, 4.79it/s] Attaching quantization: 80%|######## | 677/846 [01:47<00:40, 4.17it/s] Attaching quantization: 80%|######## | 677/846 [01:47<00:40, 4.17it/s] Attaching quantization: 80%|######## | 681/846 [01:47<00:22, 7.30it/s] Attaching quantization: 80%|######## | 681/846 [01:47<00:22, 7.30it/s] Attaching quantization: 81%|######## | 684/846 [01:47<00:16, 9.74it/s] Attaching quantization: 81%|######## | 684/846 [01:47<00:16, 9.74it/s] Attaching quantization: 81%|########1 | 686/846 [01:48<00:20, 7.93it/s] Attaching quantization: 81%|########1 | 686/846 [01:48<00:20, 7.93it/s] Attaching quantization: 81%|########1 | 688/846 [01:48<00:28, 5.45it/s] Attaching quantization: 81%|########1 | 688/846 [01:48<00:28, 5.45it/s] Attaching quantization: 82%|########1 | 690/846 [01:49<00:33, 4.65it/s] Attaching quantization: 82%|########1 | 690/846 [01:49<00:33, 4.65it/s] Attaching quantization: 82%|########1 | 691/846 [01:50<00:38, 4.04it/s] Attaching quantization: 82%|########1 | 691/846 [01:50<00:38, 4.04it/s] Attaching quantization: 82%|########2 | 695/846 [01:50<00:22, 6.73it/s] Attaching quantization: 82%|########2 | 695/846 [01:50<00:22, 6.73it/s] Attaching quantization: 83%|########2 | 698/846 [01:50<00:16, 9.12it/s] Attaching quantization: 83%|########2 | 698/846 [01:50<00:16, 9.12it/s] Attaching quantization: 83%|########2 | 700/846 [01:51<00:28, 5.19it/s] Attaching quantization: 83%|########2 | 700/846 [01:51<00:28, 5.19it/s] Attaching quantization: 83%|########2 | 702/846 [01:52<00:39, 3.61it/s] Attaching quantization: 83%|########2 | 702/846 [01:52<00:39, 3.61it/s] Attaching quantization: 83%|########3 | 703/846 [01:52<00:40, 3.57it/s] Attaching quantization: 83%|########3 | 703/846 [01:52<00:40, 3.57it/s] Attaching quantization: 83%|########3 | 704/846 [01:52<00:42, 3.35it/s] Attaching quantization: 83%|########3 | 704/846 [01:52<00:42, 3.35it/s] Attaching quantization: 83%|########3 | 705/846 [01:53<00:48, 2.93it/s] Attaching quantization: 83%|########3 | 705/846 [01:53<00:48, 2.93it/s] Attaching quantization: 84%|########3 | 710/846 [01:53<00:21, 6.27it/s] Attaching quantization: 84%|########3 | 710/846 [01:53<00:21, 6.27it/s] Attaching quantization: 84%|########4 | 712/846 [01:53<00:17, 7.48it/s] Attaching quantization: 84%|########4 | 712/846 [01:53<00:17, 7.48it/s] Attaching quantization: 84%|########4 | 714/846 [01:54<00:26, 4.93it/s] Attaching quantization: 84%|########4 | 714/846 [01:54<00:26, 4.93it/s] Attaching quantization: 85%|########4 | 716/846 [01:55<00:32, 4.05it/s] Attaching quantization: 85%|########4 | 716/846 [01:55<00:32, 4.05it/s] Attaching quantization: 85%|########4 | 717/846 [01:55<00:32, 3.95it/s] Attaching quantization: 85%|########4 | 717/846 [01:55<00:32, 3.95it/s] Attaching quantization: 85%|########4 | 718/846 [01:55<00:33, 3.81it/s] Attaching quantization: 85%|########4 | 718/846 [01:55<00:33, 3.81it/s] Attaching quantization: 85%|########4 | 719/846 [01:56<00:37, 3.41it/s] Attaching quantization: 85%|########4 | 719/846 [01:56<00:37, 3.41it/s] Attaching quantization: 86%|########5 | 724/846 [01:56<00:15, 7.78it/s] Attaching quantization: 86%|########5 | 724/846 [01:56<00:15, 7.78it/s] Attaching quantization: 86%|########6 | 728/846 [01:56<00:13, 8.66it/s] Attaching quantization: 86%|########6 | 728/846 [01:56<00:13, 8.66it/s] Attaching quantization: 86%|########6 | 730/846 [01:57<00:19, 6.01it/s] Attaching quantization: 86%|########6 | 730/846 [01:57<00:19, 6.01it/s] Attaching quantization: 87%|########6 | 732/846 [01:57<00:21, 5.27it/s] Attaching quantization: 87%|########6 | 732/846 [01:57<00:21, 5.27it/s] Attaching quantization: 87%|########6 | 733/846 [01:58<00:24, 4.59it/s] Attaching quantization: 87%|########6 | 733/846 [01:58<00:24, 4.59it/s] Attaching quantization: 87%|########7 | 737/846 [01:58<00:14, 7.72it/s] Attaching quantization: 87%|########7 | 737/846 [01:58<00:14, 7.72it/s] Attaching quantization: 87%|########7 | 740/846 [01:58<00:10, 10.21it/s] Attaching quantization: 87%|########7 | 740/846 [01:58<00:10, 10.21it/s] Attaching quantization: 88%|########7 | 743/846 [01:58<00:12, 8.19it/s] Attaching quantization: 88%|########7 | 743/846 [01:58<00:12, 8.19it/s] Attaching quantization: 88%|########8 | 745/846 [01:59<00:16, 6.12it/s] Attaching quantization: 88%|########8 | 745/846 [01:59<00:16, 6.12it/s] Attaching quantization: 88%|########8 | 747/846 [02:00<00:19, 5.02it/s] Attaching quantization: 88%|########8 | 747/846 [02:00<00:19, 5.02it/s] Attaching quantization: 89%|########8 | 751/846 [02:00<00:12, 7.48it/s] Attaching quantization: 89%|########8 | 751/846 [02:00<00:12, 7.48it/s] Attaching quantization: 89%|########9 | 753/846 [02:00<00:10, 8.71it/s] Attaching quantization: 89%|########9 | 753/846 [02:00<00:10, 8.71it/s] Attaching quantization: 89%|########9 | 756/846 [02:00<00:10, 8.92it/s] Attaching quantization: 89%|########9 | 756/846 [02:00<00:10, 8.92it/s] Attaching quantization: 90%|########9 | 758/846 [02:01<00:14, 5.91it/s] Attaching quantization: 90%|########9 | 758/846 [02:01<00:14, 5.91it/s] Attaching quantization: 90%|########9 | 760/846 [02:02<00:17, 4.99it/s] Attaching quantization: 90%|########9 | 760/846 [02:02<00:17, 4.99it/s] Attaching quantization: 90%|########9 | 761/846 [02:02<00:19, 4.26it/s] Attaching quantization: 90%|########9 | 761/846 [02:02<00:19, 4.26it/s] Attaching quantization: 90%|######### | 765/846 [02:02<00:11, 6.92it/s] Attaching quantization: 90%|######### | 765/846 [02:02<00:11, 6.92it/s] Attaching quantization: 91%|######### | 769/846 [02:02<00:07, 10.29it/s] Attaching quantization: 91%|######### | 769/846 [02:02<00:07, 10.29it/s] Attaching quantization: 91%|#########1| 771/846 [02:03<00:11, 6.52it/s] Attaching quantization: 91%|#########1| 771/846 [02:03<00:11, 6.52it/s] Attaching quantization: 91%|#########1| 773/846 [02:04<00:13, 5.25it/s] Attaching quantization: 91%|#########1| 773/846 [02:04<00:13, 5.25it/s] Attaching quantization: 92%|#########1| 775/846 [02:04<00:16, 4.29it/s] Attaching quantization: 92%|#########1| 775/846 [02:04<00:16, 4.29it/s] Attaching quantization: 92%|#########2| 779/846 [02:04<00:10, 6.55it/s] Attaching quantization: 92%|#########2| 779/846 [02:04<00:10, 6.55it/s] Attaching quantization: 92%|#########2| 781/846 [02:05<00:08, 7.73it/s] Attaching quantization: 92%|#########2| 781/846 [02:05<00:08, 7.73it/s] Attaching quantization: 93%|#########2| 783/846 [02:05<00:07, 8.25it/s] Attaching quantization: 93%|#########2| 783/846 [02:05<00:07, 8.25it/s] Attaching quantization: 93%|#########2| 785/846 [02:06<00:12, 5.03it/s] Attaching quantization: 93%|#########2| 785/846 [02:06<00:12, 5.03it/s] Attaching quantization: 93%|#########3| 787/846 [02:07<00:16, 3.61it/s] Attaching quantization: 93%|#########3| 787/846 [02:07<00:16, 3.61it/s] Attaching quantization: 93%|#########3| 788/846 [02:07<00:16, 3.53it/s] Attaching quantization: 93%|#########3| 788/846 [02:07<00:16, 3.53it/s] Attaching quantization: 93%|#########3| 789/846 [02:07<00:18, 3.13it/s] Attaching quantization: 93%|#########3| 789/846 [02:07<00:18, 3.13it/s] Attaching quantization: 94%|#########3| 793/846 [02:07<00:08, 5.97it/s] Attaching quantization: 94%|#########3| 793/846 [02:07<00:08, 5.97it/s] Attaching quantization: 94%|#########4| 797/846 [02:08<00:05, 9.31it/s] Attaching quantization: 94%|#########4| 797/846 [02:08<00:05, 9.31it/s] Attaching quantization: 95%|#########4| 800/846 [02:09<00:09, 4.94it/s] Attaching quantization: 95%|#########4| 800/846 [02:09<00:09, 4.94it/s] Attaching quantization: 95%|#########4| 802/846 [02:09<00:10, 4.27it/s] Attaching quantization: 95%|#########4| 802/846 [02:09<00:10, 4.27it/s] Attaching quantization: 95%|#########5| 804/846 [02:10<00:10, 4.04it/s] Attaching quantization: 95%|#########5| 804/846 [02:10<00:10, 4.04it/s] Attaching quantization: 96%|#########5| 808/846 [02:10<00:05, 6.40it/s] Attaching quantization: 96%|#########5| 808/846 [02:10<00:05, 6.40it/s] Attaching quantization: 96%|#########5| 811/846 [02:10<00:04, 8.34it/s] Attaching quantization: 96%|#########5| 811/846 [02:10<00:04, 8.34it/s] Attaching quantization: 96%|#########6| 814/846 [02:11<00:06, 4.88it/s] Attaching quantization: 96%|#########6| 814/846 [02:11<00:06, 4.88it/s] Attaching quantization: 96%|#########6| 816/846 [02:12<00:06, 4.43it/s] Attaching quantization: 96%|#########6| 816/846 [02:12<00:06, 4.43it/s] Attaching quantization: 97%|#########6| 818/846 [02:13<00:06, 4.49it/s] Attaching quantization: 97%|#########6| 818/846 [02:13<00:06, 4.49it/s] Attaching quantization: 97%|#########7| 821/846 [02:13<00:04, 6.07it/s] Attaching quantization: 97%|#########7| 821/846 [02:13<00:04, 6.07it/s] Attaching quantization: 98%|#########7| 825/846 [02:13<00:02, 9.00it/s] Attaching quantization: 98%|#########7| 825/846 [02:13<00:02, 9.00it/s] Attaching quantization: 98%|#########7| 827/846 [02:13<00:03, 6.13it/s] Attaching quantization: 98%|#########7| 827/846 [02:13<00:03, 6.13it/s] Attaching quantization: 98%|#########7| 829/846 [02:14<00:03, 4.77it/s] Attaching quantization: 98%|#########7| 829/846 [02:14<00:03, 4.77it/s] Attaching quantization: 98%|#########8| 831/846 [02:15<00:03, 4.07it/s] Attaching quantization: 98%|#########8| 831/846 [02:15<00:03, 4.07it/s] Attaching quantization: 99%|#########8| 835/846 [02:15<00:01, 6.24it/s] Attaching quantization: 99%|#########8| 835/846 [02:15<00:01, 6.24it/s] Attaching quantization: 99%|#########9| 839/846 [02:15<00:00, 9.06it/s] Attaching quantization: 99%|#########9| 839/846 [02:15<00:00, 9.06it/s] Attaching quantization: 99%|#########9| 841/846 [02:16<00:00, 6.21it/s] Attaching quantization: 99%|#########9| 841/846 [02:16<00:00, 6.21it/s] Attaching quantization: 100%|#########9| 843/846 [02:16<00:00, 5.15it/s] Attaching quantization: 100%|#########9| 843/846 [02:16<00:00, 5.15it/s] Attaching quantization: 100%|#########9| 845/846 [02:17<00:00, 4.86it/s] Attaching quantization: 100%|#########9| 845/846 [02:17<00:00, 4.86it/s] Attaching quantization: 100%|##########| 846/846 [02:17<00:00, 6.15it/s] Attaching quantization: 100%|##########| 846/846 [02:17<00:00, 6.15it/s] | |
| - quantizing additional layers | |
| Moving transformer to CPU | |
| Text Encoder | |
| Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s] Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s] Loading checkpoint shards: 25%|##5 | 1/4 [00:00<00:00, 6.02it/s] Loading checkpoint shards: 25%|##5 | 1/4 [00:00<00:00, 6.02it/s] Loading checkpoint shards: 100%|##########| 4/4 [00:00<00:00, 15.71it/s] Loading checkpoint shards: 100%|##########| 4/4 [00:00<00:00, 15.71it/s] | |
| Quantizing Text Encoder | |
| Loading VAE | |
| Making pipe | |
| Preparing Model | |
| Model Loaded | |
| create LoRA network. base dim (rank): 4, alpha: 4 | |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None | |
| apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16 | |
| create LoRA for Text Encoder: 0 modules. | |
| create LoRA for U-Net: 840 modules. | |
| enable LoRA for U-Net | |
| Dataset: /app/ai-toolkit/datasets/the_cosmic_drift_slider | |
| - Preprocessing image dimensions | |
| 0%| | 0/7 [00:00<?, ?it/s] 0%| | 0/7 [00:00<?, ?it/s] 43%|####2 | 3/7 [00:00<00:00, 28.41it/s] 43%|####2 | 3/7 [00:00<00:00, 28.41it/s] 86%|########5 | 6/7 [00:00<00:00, 28.66it/s] 86%|########5 | 6/7 [00:00<00:00, 28.66it/s] 100%|##########| 7/7 [00:00<00:00, 28.64it/s] 100%|##########| 7/7 [00:00<00:00, 28.64it/s] | |
| - Found 7 images | |
| - adding x axis flips | |
| - Found 14 images after adding flips | |
| Bucket sizes for /app/ai-toolkit/datasets/the_cosmic_drift_slider: | |
| 512x512: 14 files | |
| 1 buckets made | |
| ***** UNLOADING TEXT ENCODER ***** | |
| This will train only with a blank prompt or trigger word, if set | |
| If this is not what you want, remove the unload_text_encoder flag | |
| *********************************** | |
| Generating baseline samples before training | |
| Generating Images: 0%| | 0/7 [00:00<?, ?it/s] Generating Images: 0%| | 0/7 [00:00<?, ?it/s] Generating Images: 14%|#4 | 1/7 [01:35<09:33, 95.58s/it] Generating Images: 14%|#4 | 1/7 [01:35<09:33, 95.58s/it] Generating Images: 29%|##8 | 2/7 [03:11<07:57, 95.57s/it] Generating Images: 29%|##8 | 2/7 [03:11<07:57, 95.57s/it] Generating Images: 43%|####2 | 3/7 [04:46<06:22, 95.70s/it] Generating Images: 43%|####2 | 3/7 [04:46<06:22, 95.70s/it] Generating Images: 57%|#####7 | 4/7 [06:09<04:31, 90.55s/it] Generating Images: 57%|#####7 | 4/7 [06:09<04:31, 90.55s/it] Generating Images: 71%|#######1 | 5/7 [07:44<03:04, 92.27s/it] Generating Images: 71%|#######1 | 5/7 [07:44<03:04, 92.27s/it] Generating Images: 86%|########5 | 6/7 [09:20<01:33, 93.54s/it] Generating Images: 86%|########5 | 6/7 [09:20<01:33, 93.54s/it] Generating Images: 100%|##########| 7/7 [10:56<00:00, 94.12s/it] Generating Images: 100%|##########| 7/7 [10:56<00:00, 94.12s/it] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:00<?, ?it/s] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:00<?, ?it/s] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:22<?, ?it/s, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:22<?, ?it/s, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:22<?, ?it/s, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:22<?, ?it/s, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:42<?, ?it/s, lr: 1.0e-03 loss: 3.101e-02] The_Cosmic_Drift_Slider: 0%| | 0/300 [00:42<?, ?it/s, lr: 1.0e-03 loss: 3.101e-02] The_Cosmic_Drift_Slider: 0%| | 1/300 [00:42<1:44:27, 20.96s/it, lr: 1.0e-03 loss: 3.101e-02] The_Cosmic_Drift_Slider: 0%| | 1/300 [00:42<1:44:27, 20.96s/it, lr: 1.0e-03 loss: 3.101e-02] The_Cosmic_Drift_Slider: 0%| | 1/300 [01:03<1:44:27, 20.96s/it, lr: 1.0e-03 loss: 9.619e-02] The_Cosmic_Drift_Slider: 0%| | 1/300 [01:03<1:44:27, 20.96s/it, lr: 1.0e-03 loss: 9.619e-02] The_Cosmic_Drift_Slider: 1%| | 2/300 [01:03<1:43:48, 20.90s/it, lr: 1.0e-03 loss: 9.619e-02] The_Cosmic_Drift_Slider: 1%| | 2/300 [01:03<1:43:48, 20.90s/it, lr: 1.0e-03 loss: 9.619e-02] The_Cosmic_Drift_Slider: 1%| | 2/300 [01:16<1:43:48, 20.90s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 1%| | 2/300 [01:16<1:43:48, 20.90s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 1%|1 | 3/300 [01:16<1:24:55, 17.16s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 1%|1 | 3/300 [01:16<1:24:55, 17.16s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 1%|1 | 3/300 [01:30<1:24:55, 17.16s/it, lr: 1.0e-03 loss: 2.422e-01] The_Cosmic_Drift_Slider: 1%|1 | 3/300 [01:30<1:24:55, 17.16s/it, lr: 1.0e-03 loss: 2.422e-01] The_Cosmic_Drift_Slider: 1%|1 | 4/300 [01:30<1:16:17, 15.46s/it, lr: 1.0e-03 loss: 2.422e-01] The_Cosmic_Drift_Slider: 1%|1 | 4/300 [01:30<1:16:17, 15.46s/it, lr: 1.0e-03 loss: 2.422e-01] The_Cosmic_Drift_Slider: 1%|1 | 4/300 [01:50<1:16:17, 15.46s/it, lr: 1.0e-03 loss: 2.429e-02] The_Cosmic_Drift_Slider: 1%|1 | 4/300 [01:50<1:16:17, 15.46s/it, lr: 1.0e-03 loss: 2.429e-02] The_Cosmic_Drift_Slider: 2%|1 | 5/300 [01:51<1:25:30, 17.39s/it, lr: 1.0e-03 loss: 2.429e-02] The_Cosmic_Drift_Slider: 2%|1 | 5/300 [01:51<1:25:30, 17.39s/it, lr: 1.0e-03 loss: 2.429e-02] The_Cosmic_Drift_Slider: 2%|1 | 5/300 [02:11<1:25:30, 17.39s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 2%|1 | 5/300 [02:11<1:25:30, 17.39s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 2%|2 | 6/300 [02:11<1:30:55, 18.56s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 2%|2 | 6/300 [02:11<1:30:55, 18.56s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 2%|2 | 6/300 [02:32<1:30:55, 18.56s/it, lr: 1.0e-03 loss: 2.368e-02] The_Cosmic_Drift_Slider: 2%|2 | 6/300 [02:32<1:30:55, 18.56s/it, lr: 1.0e-03 loss: 2.368e-02] The_Cosmic_Drift_Slider: 2%|2 | 7/300 [02:32<1:34:15, 19.30s/it, lr: 1.0e-03 loss: 2.368e-02] The_Cosmic_Drift_Slider: 2%|2 | 7/300 [02:32<1:34:15, 19.30s/it, lr: 1.0e-03 loss: 2.368e-02] The_Cosmic_Drift_Slider: 2%|2 | 7/300 [02:54<1:34:15, 19.30s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 2%|2 | 7/300 [02:54<1:34:15, 19.30s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 3%|2 | 8/300 [02:54<1:36:32, 19.84s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 3%|2 | 8/300 [02:54<1:36:32, 19.84s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 3%|2 | 8/300 [03:15<1:36:32, 19.84s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 3%|2 | 8/300 [03:15<1:36:32, 19.84s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 3%|3 | 9/300 [03:15<1:37:41, 20.14s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 3%|3 | 9/300 [03:15<1:37:41, 20.14s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 3%|3 | 9/300 [03:27<1:37:41, 20.14s/it, lr: 1.0e-03 loss: 3.945e-01] The_Cosmic_Drift_Slider: 3%|3 | 9/300 [03:27<1:37:41, 20.14s/it, lr: 1.0e-03 loss: 3.945e-01] The_Cosmic_Drift_Slider: 3%|3 | 10/300 [03:48<1:37:21, 20.14s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 3%|3 | 10/300 [03:48<1:37:21, 20.14s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 4%|3 | 11/300 [03:48<1:14:56, 15.56s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 4%|3 | 11/300 [03:48<1:14:56, 15.56s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 4%|3 | 11/300 [04:02<1:14:56, 15.56s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 4%|3 | 11/300 [04:02<1:14:56, 15.56s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 4%|4 | 12/300 [04:02<1:11:25, 14.88s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 4%|4 | 12/300 [04:02<1:11:25, 14.88s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 4%|4 | 12/300 [04:23<1:11:25, 14.88s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 4%|4 | 12/300 [04:23<1:11:25, 14.88s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 4%|4 | 13/300 [04:23<1:18:42, 16.45s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 4%|4 | 13/300 [04:23<1:18:42, 16.45s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 4%|4 | 13/300 [04:44<1:18:42, 16.45s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 4%|4 | 13/300 [04:44<1:18:42, 16.45s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 5%|4 | 14/300 [04:44<1:24:10, 17.66s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 5%|4 | 14/300 [04:44<1:24:10, 17.66s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 5%|4 | 14/300 [05:04<1:24:10, 17.66s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 5%|4 | 14/300 [05:04<1:24:10, 17.66s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 5%|5 | 15/300 [05:04<1:27:51, 18.50s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 5%|5 | 15/300 [05:04<1:27:51, 18.50s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 5%|5 | 15/300 [05:26<1:27:51, 18.50s/it, lr: 1.0e-03 loss: 3.113e-02] The_Cosmic_Drift_Slider: 5%|5 | 15/300 [05:26<1:27:51, 18.50s/it, lr: 1.0e-03 loss: 3.113e-02] The_Cosmic_Drift_Slider: 5%|5 | 16/300 [05:26<1:30:56, 19.21s/it, lr: 1.0e-03 loss: 3.113e-02] The_Cosmic_Drift_Slider: 5%|5 | 16/300 [05:26<1:30:56, 19.21s/it, lr: 1.0e-03 loss: 3.113e-02] The_Cosmic_Drift_Slider: 5%|5 | 16/300 [05:47<1:30:56, 19.21s/it, lr: 1.0e-03 loss: 5.664e-02] The_Cosmic_Drift_Slider: 5%|5 | 16/300 [05:47<1:30:56, 19.21s/it, lr: 1.0e-03 loss: 5.664e-02] The_Cosmic_Drift_Slider: 6%|5 | 17/300 [05:47<1:32:49, 19.68s/it, lr: 1.0e-03 loss: 5.664e-02] The_Cosmic_Drift_Slider: 6%|5 | 17/300 [05:47<1:32:49, 19.68s/it, lr: 1.0e-03 loss: 5.664e-02] The_Cosmic_Drift_Slider: 6%|5 | 17/300 [06:08<1:32:49, 19.68s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 6%|5 | 17/300 [06:08<1:32:49, 19.68s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 6%|6 | 18/300 [06:08<1:34:04, 20.02s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 6%|6 | 18/300 [06:08<1:34:04, 20.02s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 6%|6 | 18/300 [06:20<1:34:04, 20.02s/it, lr: 1.0e-03 loss: 1.697e-02] The_Cosmic_Drift_Slider: 6%|6 | 18/300 [06:20<1:34:04, 20.02s/it, lr: 1.0e-03 loss: 1.697e-02] The_Cosmic_Drift_Slider: 6%|6 | 19/300 [06:20<1:23:37, 17.86s/it, lr: 1.0e-03 loss: 1.697e-02] The_Cosmic_Drift_Slider: 6%|6 | 19/300 [06:20<1:23:37, 17.86s/it, lr: 1.0e-03 loss: 1.697e-02] The_Cosmic_Drift_Slider: 6%|6 | 19/300 [06:42<1:23:37, 17.86s/it, lr: 1.0e-03 loss: 2.734e-02] The_Cosmic_Drift_Slider: 6%|6 | 19/300 [06:42<1:23:37, 17.86s/it, lr: 1.0e-03 loss: 2.734e-02] The_Cosmic_Drift_Slider: 7%|6 | 20/300 [07:03<1:23:19, 17.86s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|6 | 20/300 [07:03<1:23:19, 17.86s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|7 | 21/300 [07:03<1:07:10, 14.45s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|7 | 21/300 [07:03<1:07:10, 14.45s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|7 | 21/300 [07:16<1:07:10, 14.45s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|7 | 21/300 [07:16<1:07:10, 14.45s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|7 | 22/300 [07:16<1:05:02, 14.04s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|7 | 22/300 [07:16<1:05:02, 14.04s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 7%|7 | 22/300 [07:37<1:05:02, 14.04s/it, lr: 1.0e-03 loss: 1.680e-01] The_Cosmic_Drift_Slider: 7%|7 | 22/300 [07:37<1:05:02, 14.04s/it, lr: 1.0e-03 loss: 1.680e-01] The_Cosmic_Drift_Slider: 8%|7 | 23/300 [07:37<1:12:55, 15.80s/it, lr: 1.0e-03 loss: 1.680e-01] The_Cosmic_Drift_Slider: 8%|7 | 23/300 [07:37<1:12:55, 15.80s/it, lr: 1.0e-03 loss: 1.680e-01] The_Cosmic_Drift_Slider: 8%|7 | 23/300 [07:58<1:12:55, 15.80s/it, lr: 1.0e-03 loss: 1.260e-01] The_Cosmic_Drift_Slider: 8%|7 | 23/300 [07:58<1:12:55, 15.80s/it, lr: 1.0e-03 loss: 1.260e-01] The_Cosmic_Drift_Slider: 8%|8 | 24/300 [07:58<1:19:07, 17.20s/it, lr: 1.0e-03 loss: 1.260e-01] The_Cosmic_Drift_Slider: 8%|8 | 24/300 [07:58<1:19:07, 17.20s/it, lr: 1.0e-03 loss: 1.260e-01] The_Cosmic_Drift_Slider: 8%|8 | 24/300 [08:19<1:19:07, 17.20s/it, lr: 1.0e-03 loss: 2.197e-02] The_Cosmic_Drift_Slider: 8%|8 | 24/300 [08:19<1:19:07, 17.20s/it, lr: 1.0e-03 loss: 2.197e-02] | |
| Saving at step 25 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000025.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| The_Cosmic_Drift_Slider: 8%|8 | 25/300 [08:32<1:18:49, 17.20s/it, lr: 1.0e-03 loss: 4.199e-02] The_Cosmic_Drift_Slider: 8%|8 | 25/300 [08:32<1:18:49, 17.20s/it, lr: 1.0e-03 loss: 4.199e-02] The_Cosmic_Drift_Slider: 9%|8 | 26/300 [08:32<57:01, 12.49s/it, lr: 1.0e-03 loss: 4.199e-02] The_Cosmic_Drift_Slider: 9%|8 | 26/300 [08:32<57:01, 12.49s/it, lr: 1.0e-03 loss: 4.199e-02] The_Cosmic_Drift_Slider: 9%|8 | 26/300 [08:53<57:01, 12.49s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 9%|8 | 26/300 [08:53<57:01, 12.49s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 9%|9 | 27/300 [08:53<1:06:04, 14.52s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 9%|9 | 27/300 [08:53<1:06:04, 14.52s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 9%|9 | 27/300 [09:07<1:06:04, 14.52s/it, lr: 1.0e-03 loss: 1.495e-02] The_Cosmic_Drift_Slider: 9%|9 | 27/300 [09:07<1:06:04, 14.52s/it, lr: 1.0e-03 loss: 1.495e-02] The_Cosmic_Drift_Slider: 9%|9 | 28/300 [09:07<1:03:56, 14.11s/it, lr: 1.0e-03 loss: 1.495e-02] The_Cosmic_Drift_Slider: 9%|9 | 28/300 [09:07<1:03:56, 14.11s/it, lr: 1.0e-03 loss: 1.495e-02] The_Cosmic_Drift_Slider: 9%|9 | 28/300 [09:28<1:03:56, 14.11s/it, lr: 1.0e-03 loss: 1.240e-01] The_Cosmic_Drift_Slider: 9%|9 | 28/300 [09:28<1:03:56, 14.11s/it, lr: 1.0e-03 loss: 1.240e-01] The_Cosmic_Drift_Slider: 10%|9 | 29/300 [09:28<1:11:49, 15.90s/it, lr: 1.0e-03 loss: 1.240e-01] The_Cosmic_Drift_Slider: 10%|9 | 29/300 [09:28<1:11:49, 15.90s/it, lr: 1.0e-03 loss: 1.240e-01] The_Cosmic_Drift_Slider: 10%|9 | 29/300 [09:48<1:11:49, 15.90s/it, lr: 1.0e-03 loss: 2.441e-02] The_Cosmic_Drift_Slider: 10%|9 | 29/300 [09:48<1:11:49, 15.90s/it, lr: 1.0e-03 loss: 2.441e-02] The_Cosmic_Drift_Slider: 10%|# | 30/300 [10:09<1:11:33, 15.90s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 10%|# | 30/300 [10:09<1:11:33, 15.90s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 10%|# | 31/300 [10:09<1:00:41, 13.54s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 10%|# | 31/300 [10:09<1:00:41, 13.54s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 10%|# | 31/300 [10:31<1:00:41, 13.54s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 10%|# | 31/300 [10:31<1:00:41, 13.54s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 11%|# | 32/300 [10:31<1:08:18, 15.29s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 11%|# | 32/300 [10:31<1:08:18, 15.29s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 11%|# | 32/300 [10:52<1:08:18, 15.29s/it, lr: 1.0e-03 loss: 9.180e-02] The_Cosmic_Drift_Slider: 11%|# | 32/300 [10:52<1:08:18, 15.29s/it, lr: 1.0e-03 loss: 9.180e-02] The_Cosmic_Drift_Slider: 11%|#1 | 33/300 [10:52<1:14:14, 16.68s/it, lr: 1.0e-03 loss: 9.180e-02] The_Cosmic_Drift_Slider: 11%|#1 | 33/300 [10:52<1:14:14, 16.68s/it, lr: 1.0e-03 loss: 9.180e-02] The_Cosmic_Drift_Slider: 11%|#1 | 33/300 [11:13<1:14:14, 16.68s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 11%|#1 | 33/300 [11:13<1:14:14, 16.68s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 11%|#1 | 34/300 [11:13<1:18:49, 17.78s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 11%|#1 | 34/300 [11:13<1:18:49, 17.78s/it, lr: 1.0e-03 loss: 2.905e-02] The_Cosmic_Drift_Slider: 11%|#1 | 34/300 [11:25<1:18:49, 17.78s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 11%|#1 | 34/300 [11:25<1:18:49, 17.78s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 12%|#1 | 35/300 [11:25<1:12:24, 16.39s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 12%|#1 | 35/300 [11:25<1:12:24, 16.39s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 12%|#1 | 35/300 [11:39<1:12:24, 16.39s/it, lr: 1.0e-03 loss: 1.508e-02] The_Cosmic_Drift_Slider: 12%|#1 | 35/300 [11:39<1:12:24, 16.39s/it, lr: 1.0e-03 loss: 1.508e-02] The_Cosmic_Drift_Slider: 12%|#2 | 36/300 [11:39<1:07:49, 15.41s/it, lr: 1.0e-03 loss: 1.508e-02] The_Cosmic_Drift_Slider: 12%|#2 | 36/300 [11:39<1:07:49, 15.41s/it, lr: 1.0e-03 loss: 1.508e-02] The_Cosmic_Drift_Slider: 12%|#2 | 36/300 [12:00<1:07:49, 15.41s/it, lr: 1.0e-03 loss: 2.417e-02] The_Cosmic_Drift_Slider: 12%|#2 | 36/300 [12:00<1:07:49, 15.41s/it, lr: 1.0e-03 loss: 2.417e-02] The_Cosmic_Drift_Slider: 12%|#2 | 37/300 [12:00<1:14:11, 16.93s/it, lr: 1.0e-03 loss: 2.417e-02] The_Cosmic_Drift_Slider: 12%|#2 | 37/300 [12:00<1:14:11, 16.93s/it, lr: 1.0e-03 loss: 2.417e-02] The_Cosmic_Drift_Slider: 12%|#2 | 37/300 [12:21<1:14:11, 16.93s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 12%|#2 | 37/300 [12:21<1:14:11, 16.93s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 13%|#2 | 38/300 [12:21<1:18:54, 18.07s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 13%|#2 | 38/300 [12:21<1:18:54, 18.07s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 13%|#2 | 38/300 [12:42<1:18:54, 18.07s/it, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 13%|#2 | 38/300 [12:42<1:18:54, 18.07s/it, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 13%|#3 | 39/300 [12:42<1:22:08, 18.88s/it, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 13%|#3 | 39/300 [12:42<1:22:08, 18.88s/it, lr: 1.0e-03 loss: 3.906e-02] The_Cosmic_Drift_Slider: 13%|#3 | 39/300 [12:55<1:22:08, 18.88s/it, lr: 1.0e-03 loss: 1.440e-02] The_Cosmic_Drift_Slider: 13%|#3 | 39/300 [12:55<1:22:08, 18.88s/it, lr: 1.0e-03 loss: 1.440e-02] The_Cosmic_Drift_Slider: 13%|#3 | 40/300 [13:16<1:21:49, 18.88s/it, lr: 1.0e-03 loss: 6.055e-02] The_Cosmic_Drift_Slider: 13%|#3 | 40/300 [13:16<1:21:49, 18.88s/it, lr: 1.0e-03 loss: 6.055e-02] The_Cosmic_Drift_Slider: 14%|#3 | 41/300 [13:16<1:04:49, 15.02s/it, lr: 1.0e-03 loss: 6.055e-02] The_Cosmic_Drift_Slider: 14%|#3 | 41/300 [13:16<1:04:49, 15.02s/it, lr: 1.0e-03 loss: 6.055e-02] The_Cosmic_Drift_Slider: 14%|#3 | 41/300 [13:37<1:04:49, 15.02s/it, lr: 1.0e-03 loss: 2.295e-02] The_Cosmic_Drift_Slider: 14%|#3 | 41/300 [13:37<1:04:49, 15.02s/it, lr: 1.0e-03 loss: 2.295e-02] The_Cosmic_Drift_Slider: 14%|#4 | 42/300 [13:37<1:10:44, 16.45s/it, lr: 1.0e-03 loss: 2.295e-02] The_Cosmic_Drift_Slider: 14%|#4 | 42/300 [13:37<1:10:44, 16.45s/it, lr: 1.0e-03 loss: 2.295e-02] The_Cosmic_Drift_Slider: 14%|#4 | 42/300 [13:58<1:10:44, 16.45s/it, lr: 1.0e-03 loss: 1.660e-02] The_Cosmic_Drift_Slider: 14%|#4 | 42/300 [13:58<1:10:44, 16.45s/it, lr: 1.0e-03 loss: 1.660e-02] The_Cosmic_Drift_Slider: 14%|#4 | 43/300 [13:58<1:15:20, 17.59s/it, lr: 1.0e-03 loss: 1.660e-02] The_Cosmic_Drift_Slider: 14%|#4 | 43/300 [13:58<1:15:20, 17.59s/it, lr: 1.0e-03 loss: 1.660e-02] The_Cosmic_Drift_Slider: 14%|#4 | 43/300 [14:20<1:15:20, 17.59s/it, lr: 1.0e-03 loss: 9.668e-02] The_Cosmic_Drift_Slider: 14%|#4 | 43/300 [14:20<1:15:20, 17.59s/it, lr: 1.0e-03 loss: 9.668e-02] The_Cosmic_Drift_Slider: 15%|#4 | 44/300 [14:20<1:18:58, 18.51s/it, lr: 1.0e-03 loss: 9.668e-02] The_Cosmic_Drift_Slider: 15%|#4 | 44/300 [14:20<1:18:58, 18.51s/it, lr: 1.0e-03 loss: 9.668e-02] The_Cosmic_Drift_Slider: 15%|#4 | 44/300 [14:40<1:18:58, 18.51s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 15%|#4 | 44/300 [14:40<1:18:58, 18.51s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 15%|#5 | 45/300 [14:40<1:21:24, 19.15s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 15%|#5 | 45/300 [14:40<1:21:24, 19.15s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 15%|#5 | 45/300 [14:53<1:21:24, 19.15s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 15%|#5 | 45/300 [14:53<1:21:24, 19.15s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 15%|#5 | 46/300 [14:53<1:13:20, 17.33s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 15%|#5 | 46/300 [14:53<1:13:20, 17.33s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 15%|#5 | 46/300 [15:14<1:13:20, 17.33s/it, lr: 1.0e-03 loss: 4.834e-02] The_Cosmic_Drift_Slider: 15%|#5 | 46/300 [15:14<1:13:20, 17.33s/it, lr: 1.0e-03 loss: 4.834e-02] The_Cosmic_Drift_Slider: 16%|#5 | 47/300 [15:14<1:17:22, 18.35s/it, lr: 1.0e-03 loss: 4.834e-02] The_Cosmic_Drift_Slider: 16%|#5 | 47/300 [15:14<1:17:22, 18.35s/it, lr: 1.0e-03 loss: 4.834e-02] The_Cosmic_Drift_Slider: 16%|#5 | 47/300 [15:36<1:17:22, 18.35s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#5 | 47/300 [15:36<1:17:22, 18.35s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#6 | 48/300 [15:36<1:20:18, 19.12s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#6 | 48/300 [15:36<1:20:18, 19.12s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#6 | 48/300 [15:57<1:20:18, 19.12s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#6 | 48/300 [15:57<1:20:18, 19.12s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#6 | 49/300 [15:57<1:22:09, 19.64s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#6 | 49/300 [15:57<1:22:09, 19.64s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 16%|#6 | 49/300 [16:09<1:22:09, 19.64s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 16%|#6 | 49/300 [16:09<1:22:09, 19.64s/it, lr: 1.0e-03 loss: 2.100e-02] | |
| Saving at step 50 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000050.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| The_Cosmic_Drift_Slider: 17%|#6 | 50/300 [16:30<1:21:49, 19.64s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 17%|#6 | 50/300 [16:30<1:21:49, 19.64s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 17%|#7 | 51/300 [16:30<1:04:22, 15.51s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 17%|#7 | 51/300 [16:30<1:04:22, 15.51s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 17%|#7 | 51/300 [16:52<1:04:22, 15.51s/it, lr: 1.0e-03 loss: 6.104e-02] The_Cosmic_Drift_Slider: 17%|#7 | 51/300 [16:52<1:04:22, 15.51s/it, lr: 1.0e-03 loss: 6.104e-02] The_Cosmic_Drift_Slider: 17%|#7 | 52/300 [16:52<1:09:40, 16.86s/it, lr: 1.0e-03 loss: 6.104e-02] The_Cosmic_Drift_Slider: 17%|#7 | 52/300 [16:52<1:09:40, 16.86s/it, lr: 1.0e-03 loss: 6.104e-02] The_Cosmic_Drift_Slider: 17%|#7 | 52/300 [17:05<1:09:40, 16.86s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 17%|#7 | 52/300 [17:05<1:09:40, 16.86s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 18%|#7 | 53/300 [17:05<1:04:56, 15.78s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 18%|#7 | 53/300 [17:05<1:04:56, 15.78s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 18%|#7 | 53/300 [17:26<1:04:56, 15.78s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 18%|#7 | 53/300 [17:26<1:04:56, 15.78s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 18%|#8 | 54/300 [17:26<1:10:17, 17.14s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 18%|#8 | 54/300 [17:26<1:10:17, 17.14s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 18%|#8 | 54/300 [17:46<1:10:17, 17.14s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 18%|#8 | 54/300 [17:46<1:10:17, 17.14s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 18%|#8 | 55/300 [17:46<1:14:11, 18.17s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 18%|#8 | 55/300 [17:46<1:14:11, 18.17s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 18%|#8 | 55/300 [18:08<1:14:11, 18.17s/it, lr: 1.0e-03 loss: 4.883e-02] The_Cosmic_Drift_Slider: 18%|#8 | 55/300 [18:08<1:14:11, 18.17s/it, lr: 1.0e-03 loss: 4.883e-02] The_Cosmic_Drift_Slider: 19%|#8 | 56/300 [18:08<1:17:11, 18.98s/it, lr: 1.0e-03 loss: 4.883e-02] The_Cosmic_Drift_Slider: 19%|#8 | 56/300 [18:08<1:17:11, 18.98s/it, lr: 1.0e-03 loss: 4.883e-02] The_Cosmic_Drift_Slider: 19%|#8 | 56/300 [18:21<1:17:11, 18.98s/it, lr: 1.0e-03 loss: 1.379e-02] The_Cosmic_Drift_Slider: 19%|#8 | 56/300 [18:21<1:17:11, 18.98s/it, lr: 1.0e-03 loss: 1.379e-02] The_Cosmic_Drift_Slider: 19%|#9 | 57/300 [18:21<1:09:31, 17.17s/it, lr: 1.0e-03 loss: 1.379e-02] The_Cosmic_Drift_Slider: 19%|#9 | 57/300 [18:21<1:09:31, 17.17s/it, lr: 1.0e-03 loss: 1.379e-02] The_Cosmic_Drift_Slider: 19%|#9 | 57/300 [18:42<1:09:31, 17.17s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 19%|#9 | 57/300 [18:42<1:09:31, 17.17s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 19%|#9 | 58/300 [18:42<1:13:33, 18.24s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 19%|#9 | 58/300 [18:42<1:13:33, 18.24s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 19%|#9 | 58/300 [19:03<1:13:33, 18.24s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 19%|#9 | 58/300 [19:03<1:13:33, 18.24s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 20%|#9 | 59/300 [19:03<1:16:18, 19.00s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 20%|#9 | 59/300 [19:03<1:16:18, 19.00s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 20%|#9 | 59/300 [19:24<1:16:18, 19.00s/it, lr: 1.0e-03 loss: 2.319e-02] The_Cosmic_Drift_Slider: 20%|#9 | 59/300 [19:24<1:16:18, 19.00s/it, lr: 1.0e-03 loss: 2.319e-02] The_Cosmic_Drift_Slider: 20%|## | 60/300 [19:45<1:15:59, 19.00s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 20%|## | 60/300 [19:45<1:15:59, 19.00s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 20%|## | 61/300 [19:45<1:00:01, 15.07s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 20%|## | 61/300 [19:45<1:00:01, 15.07s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 20%|## | 61/300 [19:58<1:00:01, 15.07s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 20%|## | 61/300 [19:58<1:00:01, 15.07s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 21%|## | 62/300 [19:58<57:16, 14.44s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 21%|## | 62/300 [19:58<57:16, 14.44s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 21%|## | 62/300 [20:19<57:16, 14.44s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 21%|## | 62/300 [20:19<57:16, 14.44s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 21%|##1 | 63/300 [20:19<1:03:36, 16.10s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 21%|##1 | 63/300 [20:19<1:03:36, 16.10s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 21%|##1 | 63/300 [20:40<1:03:36, 16.10s/it, lr: 1.0e-03 loss: 1.794e-02] The_Cosmic_Drift_Slider: 21%|##1 | 63/300 [20:40<1:03:36, 16.10s/it, lr: 1.0e-03 loss: 1.794e-02] The_Cosmic_Drift_Slider: 21%|##1 | 64/300 [20:40<1:08:36, 17.44s/it, lr: 1.0e-03 loss: 1.794e-02] The_Cosmic_Drift_Slider: 21%|##1 | 64/300 [20:40<1:08:36, 17.44s/it, lr: 1.0e-03 loss: 1.794e-02] The_Cosmic_Drift_Slider: 21%|##1 | 64/300 [21:01<1:08:36, 17.44s/it, lr: 1.0e-03 loss: 5.103e-02] The_Cosmic_Drift_Slider: 21%|##1 | 64/300 [21:01<1:08:36, 17.44s/it, lr: 1.0e-03 loss: 5.103e-02] The_Cosmic_Drift_Slider: 22%|##1 | 65/300 [21:01<1:12:01, 18.39s/it, lr: 1.0e-03 loss: 5.103e-02] The_Cosmic_Drift_Slider: 22%|##1 | 65/300 [21:01<1:12:01, 18.39s/it, lr: 1.0e-03 loss: 5.103e-02] The_Cosmic_Drift_Slider: 22%|##1 | 65/300 [21:14<1:12:01, 18.39s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 22%|##1 | 65/300 [21:14<1:12:01, 18.39s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 22%|##2 | 66/300 [21:14<1:05:27, 16.79s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 22%|##2 | 66/300 [21:14<1:05:27, 16.79s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 22%|##2 | 66/300 [21:35<1:05:27, 16.79s/it, lr: 1.0e-03 loss: 2.722e-02] The_Cosmic_Drift_Slider: 22%|##2 | 66/300 [21:35<1:05:27, 16.79s/it, lr: 1.0e-03 loss: 2.722e-02] The_Cosmic_Drift_Slider: 22%|##2 | 67/300 [21:35<1:09:43, 17.95s/it, lr: 1.0e-03 loss: 2.722e-02] The_Cosmic_Drift_Slider: 22%|##2 | 67/300 [21:35<1:09:43, 17.95s/it, lr: 1.0e-03 loss: 2.722e-02] The_Cosmic_Drift_Slider: 22%|##2 | 67/300 [21:49<1:09:43, 17.95s/it, lr: 1.0e-03 loss: 3.027e-02] The_Cosmic_Drift_Slider: 22%|##2 | 67/300 [21:49<1:09:43, 17.95s/it, lr: 1.0e-03 loss: 3.027e-02] The_Cosmic_Drift_Slider: 23%|##2 | 68/300 [21:49<1:03:41, 16.47s/it, lr: 1.0e-03 loss: 3.027e-02] The_Cosmic_Drift_Slider: 23%|##2 | 68/300 [21:49<1:03:41, 16.47s/it, lr: 1.0e-03 loss: 3.027e-02] The_Cosmic_Drift_Slider: 23%|##2 | 68/300 [22:09<1:03:41, 16.47s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 23%|##2 | 68/300 [22:09<1:03:41, 16.47s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 23%|##3 | 69/300 [22:09<1:08:22, 17.76s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 23%|##3 | 69/300 [22:09<1:08:22, 17.76s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 23%|##3 | 69/300 [22:30<1:08:22, 17.76s/it, lr: 1.0e-03 loss: 2.319e-02] The_Cosmic_Drift_Slider: 23%|##3 | 69/300 [22:30<1:08:22, 17.76s/it, lr: 1.0e-03 loss: 2.319e-02] The_Cosmic_Drift_Slider: 23%|##3 | 70/300 [22:51<1:08:04, 17.76s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 23%|##3 | 70/300 [22:51<1:08:04, 17.76s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 24%|##3 | 71/300 [22:51<54:57, 14.40s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 24%|##3 | 71/300 [22:51<54:57, 14.40s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 24%|##3 | 71/300 [23:05<54:57, 14.40s/it, lr: 1.0e-03 loss: 1.331e-02] The_Cosmic_Drift_Slider: 24%|##3 | 71/300 [23:05<54:57, 14.40s/it, lr: 1.0e-03 loss: 1.331e-02] The_Cosmic_Drift_Slider: 24%|##4 | 72/300 [23:05<53:17, 14.03s/it, lr: 1.0e-03 loss: 1.331e-02] The_Cosmic_Drift_Slider: 24%|##4 | 72/300 [23:05<53:17, 14.03s/it, lr: 1.0e-03 loss: 1.331e-02] The_Cosmic_Drift_Slider: 24%|##4 | 72/300 [23:26<53:17, 14.03s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 24%|##4 | 72/300 [23:26<53:17, 14.03s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 24%|##4 | 73/300 [23:26<59:45, 15.79s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 24%|##4 | 73/300 [23:26<59:45, 15.79s/it, lr: 1.0e-03 loss: 2.209e-02] The_Cosmic_Drift_Slider: 24%|##4 | 73/300 [23:46<59:45, 15.79s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 24%|##4 | 73/300 [23:46<59:45, 15.79s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 25%|##4 | 74/300 [23:46<1:04:39, 17.17s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 25%|##4 | 74/300 [23:46<1:04:39, 17.17s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 25%|##4 | 74/300 [24:07<1:04:39, 17.17s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 25%|##4 | 74/300 [24:07<1:04:39, 17.17s/it, lr: 1.0e-03 loss: 3.198e-02] | |
| Saving at step 75 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000075.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| The_Cosmic_Drift_Slider: 25%|##5 | 75/300 [24:21<1:04:22, 17.17s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 25%|##5 | 75/300 [24:21<1:04:22, 17.17s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 25%|##5 | 76/300 [24:21<46:23, 12.43s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 25%|##5 | 76/300 [24:21<46:23, 12.43s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 25%|##5 | 76/300 [24:42<46:23, 12.43s/it, lr: 1.0e-03 loss: 8.740e-02] The_Cosmic_Drift_Slider: 25%|##5 | 76/300 [24:42<46:23, 12.43s/it, lr: 1.0e-03 loss: 8.740e-02] The_Cosmic_Drift_Slider: 26%|##5 | 77/300 [24:42<53:53, 14.50s/it, lr: 1.0e-03 loss: 8.740e-02] The_Cosmic_Drift_Slider: 26%|##5 | 77/300 [24:42<53:53, 14.50s/it, lr: 1.0e-03 loss: 8.740e-02] The_Cosmic_Drift_Slider: 26%|##5 | 77/300 [25:03<53:53, 14.50s/it, lr: 1.0e-03 loss: 3.613e-02] The_Cosmic_Drift_Slider: 26%|##5 | 77/300 [25:03<53:53, 14.50s/it, lr: 1.0e-03 loss: 3.613e-02] The_Cosmic_Drift_Slider: 26%|##6 | 78/300 [25:03<59:34, 16.10s/it, lr: 1.0e-03 loss: 3.613e-02] The_Cosmic_Drift_Slider: 26%|##6 | 78/300 [25:03<59:34, 16.10s/it, lr: 1.0e-03 loss: 3.613e-02] The_Cosmic_Drift_Slider: 26%|##6 | 78/300 [25:23<59:34, 16.10s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 26%|##6 | 78/300 [25:23<59:34, 16.10s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 26%|##6 | 79/300 [25:23<1:03:55, 17.36s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 26%|##6 | 79/300 [25:23<1:03:55, 17.36s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 26%|##6 | 79/300 [25:45<1:03:55, 17.36s/it, lr: 1.0e-03 loss: 1.843e-02] The_Cosmic_Drift_Slider: 26%|##6 | 79/300 [25:45<1:03:55, 17.36s/it, lr: 1.0e-03 loss: 1.843e-02] The_Cosmic_Drift_Slider: 27%|##6 | 80/300 [25:58<1:03:38, 17.36s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 27%|##6 | 80/300 [25:58<1:03:38, 17.36s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 27%|##7 | 81/300 [25:58<46:02, 12.61s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 27%|##7 | 81/300 [25:58<46:02, 12.61s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 27%|##7 | 81/300 [26:19<46:02, 12.61s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 27%|##7 | 81/300 [26:19<46:02, 12.61s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 27%|##7 | 82/300 [26:19<52:55, 14.57s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 27%|##7 | 82/300 [26:19<52:55, 14.57s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 27%|##7 | 82/300 [26:40<52:55, 14.57s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 27%|##7 | 82/300 [26:40<52:55, 14.57s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 28%|##7 | 83/300 [26:40<58:24, 16.15s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 28%|##7 | 83/300 [26:40<58:24, 16.15s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 28%|##7 | 83/300 [27:02<58:24, 16.15s/it, lr: 1.0e-03 loss: 1.641e-01] The_Cosmic_Drift_Slider: 28%|##7 | 83/300 [27:02<58:24, 16.15s/it, lr: 1.0e-03 loss: 1.641e-01] The_Cosmic_Drift_Slider: 28%|##8 | 84/300 [27:02<1:02:44, 17.43s/it, lr: 1.0e-03 loss: 1.641e-01] The_Cosmic_Drift_Slider: 28%|##8 | 84/300 [27:02<1:02:44, 17.43s/it, lr: 1.0e-03 loss: 1.641e-01] The_Cosmic_Drift_Slider: 28%|##8 | 84/300 [27:22<1:02:44, 17.43s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 28%|##8 | 84/300 [27:22<1:02:44, 17.43s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 28%|##8 | 85/300 [27:22<1:05:47, 18.36s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 28%|##8 | 85/300 [27:22<1:05:47, 18.36s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 28%|##8 | 85/300 [27:35<1:05:47, 18.36s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 28%|##8 | 85/300 [27:35<1:05:47, 18.36s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 29%|##8 | 86/300 [27:35<59:49, 16.77s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 29%|##8 | 86/300 [27:35<59:49, 16.77s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 29%|##8 | 86/300 [27:56<59:49, 16.77s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 29%|##8 | 86/300 [27:56<59:49, 16.77s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 29%|##9 | 87/300 [27:56<1:03:28, 17.88s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 29%|##9 | 87/300 [27:56<1:03:28, 17.88s/it, lr: 1.0e-03 loss: 2.100e-02] The_Cosmic_Drift_Slider: 29%|##9 | 87/300 [28:18<1:03:28, 17.88s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 29%|##9 | 87/300 [28:18<1:03:28, 17.88s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 29%|##9 | 88/300 [28:18<1:06:23, 18.79s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 29%|##9 | 88/300 [28:18<1:06:23, 18.79s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 29%|##9 | 88/300 [28:38<1:06:23, 18.79s/it, lr: 1.0e-03 loss: 2.197e-02] The_Cosmic_Drift_Slider: 29%|##9 | 88/300 [28:38<1:06:23, 18.79s/it, lr: 1.0e-03 loss: 2.197e-02] The_Cosmic_Drift_Slider: 30%|##9 | 89/300 [28:38<1:08:12, 19.39s/it, lr: 1.0e-03 loss: 2.197e-02] The_Cosmic_Drift_Slider: 30%|##9 | 89/300 [28:38<1:08:12, 19.39s/it, lr: 1.0e-03 loss: 2.197e-02] The_Cosmic_Drift_Slider: 30%|##9 | 89/300 [28:51<1:08:12, 19.39s/it, lr: 1.0e-03 loss: 1.160e-02] The_Cosmic_Drift_Slider: 30%|##9 | 89/300 [28:51<1:08:12, 19.39s/it, lr: 1.0e-03 loss: 1.160e-02] The_Cosmic_Drift_Slider: 30%|### | 90/300 [29:12<1:07:52, 19.39s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 30%|### | 90/300 [29:12<1:07:52, 19.39s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 30%|### | 91/300 [29:12<53:18, 15.31s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 30%|### | 91/300 [29:12<53:18, 15.31s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 30%|### | 91/300 [29:26<53:18, 15.31s/it, lr: 1.0e-03 loss: 1.221e-02] The_Cosmic_Drift_Slider: 30%|### | 91/300 [29:26<53:18, 15.31s/it, lr: 1.0e-03 loss: 1.221e-02] The_Cosmic_Drift_Slider: 31%|### | 92/300 [29:26<51:01, 14.72s/it, lr: 1.0e-03 loss: 1.221e-02] The_Cosmic_Drift_Slider: 31%|### | 92/300 [29:26<51:01, 14.72s/it, lr: 1.0e-03 loss: 1.221e-02] The_Cosmic_Drift_Slider: 31%|### | 92/300 [29:46<51:01, 14.72s/it, lr: 1.0e-03 loss: 8.984e-02] The_Cosmic_Drift_Slider: 31%|### | 92/300 [29:46<51:01, 14.72s/it, lr: 1.0e-03 loss: 8.984e-02] The_Cosmic_Drift_Slider: 31%|###1 | 93/300 [29:47<56:15, 16.31s/it, lr: 1.0e-03 loss: 8.984e-02] The_Cosmic_Drift_Slider: 31%|###1 | 93/300 [29:47<56:15, 16.31s/it, lr: 1.0e-03 loss: 8.984e-02] The_Cosmic_Drift_Slider: 31%|###1 | 93/300 [30:07<56:15, 16.31s/it, lr: 1.0e-03 loss: 2.991e-02] The_Cosmic_Drift_Slider: 31%|###1 | 93/300 [30:07<56:15, 16.31s/it, lr: 1.0e-03 loss: 2.991e-02] The_Cosmic_Drift_Slider: 31%|###1 | 94/300 [30:07<1:00:10, 17.53s/it, lr: 1.0e-03 loss: 2.991e-02] The_Cosmic_Drift_Slider: 31%|###1 | 94/300 [30:07<1:00:10, 17.53s/it, lr: 1.0e-03 loss: 2.991e-02] The_Cosmic_Drift_Slider: 31%|###1 | 94/300 [30:28<1:00:10, 17.53s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 31%|###1 | 94/300 [30:28<1:00:10, 17.53s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 32%|###1 | 95/300 [30:28<1:03:04, 18.46s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 32%|###1 | 95/300 [30:28<1:03:04, 18.46s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 32%|###1 | 95/300 [30:42<1:03:04, 18.46s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 32%|###1 | 95/300 [30:42<1:03:04, 18.46s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 32%|###2 | 96/300 [30:42<57:23, 16.88s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 32%|###2 | 96/300 [30:42<57:23, 16.88s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 32%|###2 | 96/300 [31:03<57:23, 16.88s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 32%|###2 | 96/300 [31:03<57:23, 16.88s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 32%|###2 | 97/300 [31:03<1:00:59, 18.03s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 32%|###2 | 97/300 [31:03<1:00:59, 18.03s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 32%|###2 | 97/300 [31:24<1:00:59, 18.03s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 32%|###2 | 97/300 [31:24<1:00:59, 18.03s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 33%|###2 | 98/300 [31:24<1:03:30, 18.86s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 33%|###2 | 98/300 [31:24<1:03:30, 18.86s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 33%|###2 | 98/300 [31:44<1:03:30, 18.86s/it, lr: 1.0e-03 loss: 1.636e-02] The_Cosmic_Drift_Slider: 33%|###2 | 98/300 [31:44<1:03:30, 18.86s/it, lr: 1.0e-03 loss: 1.636e-02] The_Cosmic_Drift_Slider: 33%|###3 | 99/300 [31:44<1:05:09, 19.45s/it, lr: 1.0e-03 loss: 1.636e-02] The_Cosmic_Drift_Slider: 33%|###3 | 99/300 [31:44<1:05:09, 19.45s/it, lr: 1.0e-03 loss: 1.636e-02] The_Cosmic_Drift_Slider: 33%|###3 | 99/300 [32:06<1:05:09, 19.45s/it, lr: 1.0e-03 loss: 1.660e-02] The_Cosmic_Drift_Slider: 33%|###3 | 99/300 [32:06<1:05:09, 19.45s/it, lr: 1.0e-03 loss: 1.660e-02] | |
| Saving at step 100 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000100.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| The_Cosmic_Drift_Slider: 33%|###3 | 100/300 [32:28<1:04:50, 19.45s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 33%|###3 | 100/300 [32:28<1:04:50, 19.45s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 34%|###3 | 101/300 [32:28<51:34, 15.55s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 34%|###3 | 101/300 [32:28<51:34, 15.55s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 34%|###3 | 101/300 [32:41<51:34, 15.55s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 34%|###3 | 101/300 [32:41<51:34, 15.55s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 34%|###4 | 102/300 [32:41<49:02, 14.86s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 34%|###4 | 102/300 [32:41<49:02, 14.86s/it, lr: 1.0e-03 loss: 7.812e-02] The_Cosmic_Drift_Slider: 34%|###4 | 102/300 [33:01<49:02, 14.86s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 34%|###4 | 102/300 [33:01<49:02, 14.86s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 34%|###4 | 103/300 [33:01<53:52, 16.41s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 34%|###4 | 103/300 [33:01<53:52, 16.41s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 34%|###4 | 103/300 [33:23<53:52, 16.41s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 34%|###4 | 103/300 [33:23<53:52, 16.41s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 35%|###4 | 104/300 [33:23<57:39, 17.65s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 35%|###4 | 104/300 [33:23<57:39, 17.65s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 35%|###4 | 104/300 [33:36<57:39, 17.65s/it, lr: 1.0e-03 loss: 2.478e-02] The_Cosmic_Drift_Slider: 35%|###4 | 104/300 [33:36<57:39, 17.65s/it, lr: 1.0e-03 loss: 2.478e-02] The_Cosmic_Drift_Slider: 35%|###5 | 105/300 [33:36<52:54, 16.28s/it, lr: 1.0e-03 loss: 2.478e-02] The_Cosmic_Drift_Slider: 35%|###5 | 105/300 [33:36<52:54, 16.28s/it, lr: 1.0e-03 loss: 2.478e-02] The_Cosmic_Drift_Slider: 35%|###5 | 105/300 [33:57<52:54, 16.28s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 35%|###5 | 105/300 [33:57<52:54, 16.28s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 35%|###5 | 106/300 [33:57<56:51, 17.58s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 35%|###5 | 106/300 [33:57<56:51, 17.58s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 35%|###5 | 106/300 [34:18<56:51, 17.58s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 35%|###5 | 106/300 [34:18<56:51, 17.58s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 36%|###5 | 107/300 [34:18<59:36, 18.53s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 36%|###5 | 107/300 [34:18<59:36, 18.53s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 36%|###5 | 107/300 [34:39<59:36, 18.53s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 36%|###5 | 107/300 [34:39<59:36, 18.53s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 36%|###6 | 108/300 [34:39<1:01:35, 19.25s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 36%|###6 | 108/300 [34:39<1:01:35, 19.25s/it, lr: 1.0e-03 loss: 1.904e-02] The_Cosmic_Drift_Slider: 36%|###6 | 108/300 [35:00<1:01:35, 19.25s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 36%|###6 | 108/300 [35:00<1:01:35, 19.25s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 36%|###6 | 109/300 [35:00<1:02:43, 19.71s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 36%|###6 | 109/300 [35:00<1:02:43, 19.71s/it, lr: 1.0e-03 loss: 2.539e-02] The_Cosmic_Drift_Slider: 36%|###6 | 109/300 [35:13<1:02:43, 19.71s/it, lr: 1.0e-03 loss: 1.556e-02] The_Cosmic_Drift_Slider: 36%|###6 | 109/300 [35:13<1:02:43, 19.71s/it, lr: 1.0e-03 loss: 1.556e-02] The_Cosmic_Drift_Slider: 37%|###6 | 110/300 [35:34<1:02:24, 19.71s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 37%|###6 | 110/300 [35:34<1:02:24, 19.71s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 37%|###7 | 111/300 [35:34<48:41, 15.46s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 37%|###7 | 111/300 [35:34<48:41, 15.46s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 37%|###7 | 111/300 [35:47<48:41, 15.46s/it, lr: 1.0e-03 loss: 7.275e-02] The_Cosmic_Drift_Slider: 37%|###7 | 111/300 [35:47<48:41, 15.46s/it, lr: 1.0e-03 loss: 7.275e-02] The_Cosmic_Drift_Slider: 37%|###7 | 112/300 [35:47<46:12, 14.75s/it, lr: 1.0e-03 loss: 7.275e-02] The_Cosmic_Drift_Slider: 37%|###7 | 112/300 [35:47<46:12, 14.75s/it, lr: 1.0e-03 loss: 7.275e-02] The_Cosmic_Drift_Slider: 37%|###7 | 112/300 [36:08<46:12, 14.75s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 37%|###7 | 112/300 [36:08<46:12, 14.75s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 38%|###7 | 113/300 [36:08<50:58, 16.36s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 38%|###7 | 113/300 [36:08<50:58, 16.36s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 38%|###7 | 113/300 [36:29<50:58, 16.36s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 38%|###7 | 113/300 [36:29<50:58, 16.36s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 38%|###8 | 114/300 [36:29<54:26, 17.56s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 38%|###8 | 114/300 [36:29<54:26, 17.56s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 38%|###8 | 114/300 [36:50<54:26, 17.56s/it, lr: 1.0e-03 loss: 4.248e-02] The_Cosmic_Drift_Slider: 38%|###8 | 114/300 [36:50<54:26, 17.56s/it, lr: 1.0e-03 loss: 4.248e-02] The_Cosmic_Drift_Slider: 38%|###8 | 115/300 [36:50<56:57, 18.47s/it, lr: 1.0e-03 loss: 4.248e-02] The_Cosmic_Drift_Slider: 38%|###8 | 115/300 [36:50<56:57, 18.47s/it, lr: 1.0e-03 loss: 4.248e-02] The_Cosmic_Drift_Slider: 38%|###8 | 115/300 [37:11<56:57, 18.47s/it, lr: 1.0e-03 loss: 6.445e-02] The_Cosmic_Drift_Slider: 38%|###8 | 115/300 [37:11<56:57, 18.47s/it, lr: 1.0e-03 loss: 6.445e-02] The_Cosmic_Drift_Slider: 39%|###8 | 116/300 [37:11<58:52, 19.20s/it, lr: 1.0e-03 loss: 6.445e-02] The_Cosmic_Drift_Slider: 39%|###8 | 116/300 [37:11<58:52, 19.20s/it, lr: 1.0e-03 loss: 6.445e-02] The_Cosmic_Drift_Slider: 39%|###8 | 116/300 [37:24<58:52, 19.20s/it, lr: 1.0e-03 loss: 1.672e-02] The_Cosmic_Drift_Slider: 39%|###8 | 116/300 [37:24<58:52, 19.20s/it, lr: 1.0e-03 loss: 1.672e-02] The_Cosmic_Drift_Slider: 39%|###9 | 117/300 [37:24<52:50, 17.32s/it, lr: 1.0e-03 loss: 1.672e-02] The_Cosmic_Drift_Slider: 39%|###9 | 117/300 [37:24<52:50, 17.32s/it, lr: 1.0e-03 loss: 1.672e-02] The_Cosmic_Drift_Slider: 39%|###9 | 117/300 [37:45<52:50, 17.32s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 39%|###9 | 117/300 [37:45<52:50, 17.32s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 39%|###9 | 118/300 [37:45<55:39, 18.35s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 39%|###9 | 118/300 [37:45<55:39, 18.35s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 39%|###9 | 118/300 [38:06<55:39, 18.35s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 39%|###9 | 118/300 [38:06<55:39, 18.35s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 40%|###9 | 119/300 [38:06<57:32, 19.07s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 40%|###9 | 119/300 [38:06<57:32, 19.07s/it, lr: 1.0e-03 loss: 3.516e-02] The_Cosmic_Drift_Slider: 40%|###9 | 119/300 [38:28<57:32, 19.07s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 40%|###9 | 119/300 [38:28<57:32, 19.07s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 40%|#### | 120/300 [38:48<57:13, 19.07s/it, lr: 1.0e-03 loss: 7.959e-02] The_Cosmic_Drift_Slider: 40%|#### | 120/300 [38:48<57:13, 19.07s/it, lr: 1.0e-03 loss: 7.959e-02] The_Cosmic_Drift_Slider: 40%|#### | 121/300 [38:48<45:04, 15.11s/it, lr: 1.0e-03 loss: 7.959e-02] The_Cosmic_Drift_Slider: 40%|#### | 121/300 [38:48<45:04, 15.11s/it, lr: 1.0e-03 loss: 7.959e-02] The_Cosmic_Drift_Slider: 40%|#### | 121/300 [39:01<45:04, 15.11s/it, lr: 1.0e-03 loss: 8.594e-02] The_Cosmic_Drift_Slider: 40%|#### | 121/300 [39:01<45:04, 15.11s/it, lr: 1.0e-03 loss: 8.594e-02] The_Cosmic_Drift_Slider: 41%|#### | 122/300 [39:01<43:04, 14.52s/it, lr: 1.0e-03 loss: 8.594e-02] The_Cosmic_Drift_Slider: 41%|#### | 122/300 [39:01<43:04, 14.52s/it, lr: 1.0e-03 loss: 8.594e-02] The_Cosmic_Drift_Slider: 41%|#### | 122/300 [39:22<43:04, 14.52s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 41%|#### | 122/300 [39:22<43:04, 14.52s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 41%|####1 | 123/300 [39:22<47:40, 16.16s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 41%|####1 | 123/300 [39:22<47:40, 16.16s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 41%|####1 | 123/300 [39:44<47:40, 16.16s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 41%|####1 | 123/300 [39:44<47:40, 16.16s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 41%|####1 | 124/300 [39:44<51:15, 17.48s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 41%|####1 | 124/300 [39:44<51:15, 17.48s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 41%|####1 | 124/300 [40:04<51:15, 17.48s/it, lr: 1.0e-03 loss: 2.197e-02] The_Cosmic_Drift_Slider: 41%|####1 | 124/300 [40:04<51:15, 17.48s/it, lr: 1.0e-03 loss: 2.197e-02] | |
| Saving at step 125 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000125.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| Removing old save: /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000025.safetensors | |
| The_Cosmic_Drift_Slider: 42%|####1 | 125/300 [40:18<50:58, 17.48s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 42%|####1 | 125/300 [40:18<50:58, 17.48s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 42%|####2 | 126/300 [40:18<37:00, 12.76s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 42%|####2 | 126/300 [40:18<37:00, 12.76s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 42%|####2 | 126/300 [40:39<37:00, 12.76s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 42%|####2 | 126/300 [40:39<37:00, 12.76s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 42%|####2 | 127/300 [40:39<42:18, 14.67s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 42%|####2 | 127/300 [40:39<42:18, 14.67s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 42%|####2 | 127/300 [40:52<42:18, 14.67s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 42%|####2 | 127/300 [40:52<42:18, 14.67s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 43%|####2 | 128/300 [40:52<40:49, 14.24s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 43%|####2 | 128/300 [40:52<40:49, 14.24s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 43%|####2 | 128/300 [41:13<40:49, 14.24s/it, lr: 1.0e-03 loss: 1.318e-02] The_Cosmic_Drift_Slider: 43%|####2 | 128/300 [41:13<40:49, 14.24s/it, lr: 1.0e-03 loss: 1.318e-02] The_Cosmic_Drift_Slider: 43%|####3 | 129/300 [41:13<45:35, 16.00s/it, lr: 1.0e-03 loss: 1.318e-02] The_Cosmic_Drift_Slider: 43%|####3 | 129/300 [41:13<45:35, 16.00s/it, lr: 1.0e-03 loss: 1.318e-02] The_Cosmic_Drift_Slider: 43%|####3 | 129/300 [41:34<45:35, 16.00s/it, lr: 1.0e-03 loss: 2.429e-02] The_Cosmic_Drift_Slider: 43%|####3 | 129/300 [41:34<45:35, 16.00s/it, lr: 1.0e-03 loss: 2.429e-02] The_Cosmic_Drift_Slider: 43%|####3 | 130/300 [41:55<45:19, 16.00s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 43%|####3 | 130/300 [41:55<45:19, 16.00s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 44%|####3 | 131/300 [41:55<38:17, 13.59s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 44%|####3 | 131/300 [41:55<38:17, 13.59s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 44%|####3 | 131/300 [42:09<38:17, 13.59s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 44%|####3 | 131/300 [42:09<38:17, 13.59s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 44%|####4 | 132/300 [42:09<37:35, 13.43s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 44%|####4 | 132/300 [42:09<37:35, 13.43s/it, lr: 1.0e-03 loss: 6.836e-02] The_Cosmic_Drift_Slider: 44%|####4 | 132/300 [42:29<37:35, 13.43s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 44%|####4 | 132/300 [42:29<37:35, 13.43s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 44%|####4 | 133/300 [42:29<42:33, 15.29s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 44%|####4 | 133/300 [42:29<42:33, 15.29s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 44%|####4 | 133/300 [42:50<42:33, 15.29s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 44%|####4 | 133/300 [42:50<42:33, 15.29s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 45%|####4 | 134/300 [42:50<46:20, 16.75s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 45%|####4 | 134/300 [42:50<46:20, 16.75s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 45%|####4 | 134/300 [43:11<46:20, 16.75s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 45%|####4 | 134/300 [43:11<46:20, 16.75s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 45%|####5 | 135/300 [43:11<49:08, 17.87s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 45%|####5 | 135/300 [43:11<49:08, 17.87s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 45%|####5 | 135/300 [43:25<49:08, 17.87s/it, lr: 1.0e-03 loss: 1.337e-02] The_Cosmic_Drift_Slider: 45%|####5 | 135/300 [43:25<49:08, 17.87s/it, lr: 1.0e-03 loss: 1.337e-02] The_Cosmic_Drift_Slider: 45%|####5 | 136/300 [43:25<45:01, 16.47s/it, lr: 1.0e-03 loss: 1.337e-02] The_Cosmic_Drift_Slider: 45%|####5 | 136/300 [43:25<45:01, 16.47s/it, lr: 1.0e-03 loss: 1.337e-02] The_Cosmic_Drift_Slider: 45%|####5 | 136/300 [43:45<45:01, 16.47s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 45%|####5 | 136/300 [43:45<45:01, 16.47s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 46%|####5 | 137/300 [43:45<48:00, 17.67s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 46%|####5 | 137/300 [43:45<48:00, 17.67s/it, lr: 1.0e-03 loss: 1.465e-02] The_Cosmic_Drift_Slider: 46%|####5 | 137/300 [44:06<48:00, 17.67s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 46%|####5 | 137/300 [44:06<48:00, 17.67s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 46%|####6 | 138/300 [44:06<50:12, 18.60s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 46%|####6 | 138/300 [44:06<50:12, 18.60s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 46%|####6 | 138/300 [44:27<50:12, 18.60s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 46%|####6 | 138/300 [44:27<50:12, 18.60s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 46%|####6 | 139/300 [44:27<51:40, 19.26s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 46%|####6 | 139/300 [44:27<51:40, 19.26s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 46%|####6 | 139/300 [44:49<51:40, 19.26s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 46%|####6 | 139/300 [44:49<51:40, 19.26s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 47%|####6 | 140/300 [45:10<51:21, 19.26s/it, lr: 1.0e-03 loss: 3.247e-02] The_Cosmic_Drift_Slider: 47%|####6 | 140/300 [45:10<51:21, 19.26s/it, lr: 1.0e-03 loss: 3.247e-02] The_Cosmic_Drift_Slider: 47%|####6 | 141/300 [45:10<40:21, 15.23s/it, lr: 1.0e-03 loss: 3.247e-02] The_Cosmic_Drift_Slider: 47%|####6 | 141/300 [45:10<40:21, 15.23s/it, lr: 1.0e-03 loss: 3.247e-02] The_Cosmic_Drift_Slider: 47%|####6 | 141/300 [45:31<40:21, 15.23s/it, lr: 1.0e-03 loss: 1.182e-01] The_Cosmic_Drift_Slider: 47%|####6 | 141/300 [45:31<40:21, 15.23s/it, lr: 1.0e-03 loss: 1.182e-01] The_Cosmic_Drift_Slider: 47%|####7 | 142/300 [45:31<43:45, 16.61s/it, lr: 1.0e-03 loss: 1.182e-01] The_Cosmic_Drift_Slider: 47%|####7 | 142/300 [45:31<43:45, 16.61s/it, lr: 1.0e-03 loss: 1.182e-01] The_Cosmic_Drift_Slider: 47%|####7 | 142/300 [45:43<43:45, 16.61s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 47%|####7 | 142/300 [45:43<43:45, 16.61s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 48%|####7 | 143/300 [45:43<40:49, 15.60s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 48%|####7 | 143/300 [45:43<40:49, 15.60s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 48%|####7 | 143/300 [46:05<40:49, 15.60s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 48%|####7 | 143/300 [46:05<40:49, 15.60s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 48%|####8 | 144/300 [46:05<44:22, 17.07s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 48%|####8 | 144/300 [46:05<44:22, 17.07s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 48%|####8 | 144/300 [46:26<44:22, 17.07s/it, lr: 1.0e-03 loss: 3.076e-02] The_Cosmic_Drift_Slider: 48%|####8 | 144/300 [46:26<44:22, 17.07s/it, lr: 1.0e-03 loss: 3.076e-02] The_Cosmic_Drift_Slider: 48%|####8 | 145/300 [46:26<46:49, 18.13s/it, lr: 1.0e-03 loss: 3.076e-02] The_Cosmic_Drift_Slider: 48%|####8 | 145/300 [46:26<46:49, 18.13s/it, lr: 1.0e-03 loss: 3.076e-02] The_Cosmic_Drift_Slider: 48%|####8 | 145/300 [46:39<46:49, 18.13s/it, lr: 1.0e-03 loss: 6.152e-02] The_Cosmic_Drift_Slider: 48%|####8 | 145/300 [46:39<46:49, 18.13s/it, lr: 1.0e-03 loss: 6.152e-02] The_Cosmic_Drift_Slider: 49%|####8 | 146/300 [46:39<42:35, 16.59s/it, lr: 1.0e-03 loss: 6.152e-02] The_Cosmic_Drift_Slider: 49%|####8 | 146/300 [46:39<42:35, 16.59s/it, lr: 1.0e-03 loss: 6.152e-02] The_Cosmic_Drift_Slider: 49%|####8 | 146/300 [47:00<42:35, 16.59s/it, lr: 1.0e-03 loss: 7.520e-02] The_Cosmic_Drift_Slider: 49%|####8 | 146/300 [47:00<42:35, 16.59s/it, lr: 1.0e-03 loss: 7.520e-02] The_Cosmic_Drift_Slider: 49%|####9 | 147/300 [47:00<45:25, 17.82s/it, lr: 1.0e-03 loss: 7.520e-02] The_Cosmic_Drift_Slider: 49%|####9 | 147/300 [47:00<45:25, 17.82s/it, lr: 1.0e-03 loss: 7.520e-02] The_Cosmic_Drift_Slider: 49%|####9 | 147/300 [47:21<45:25, 17.82s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 49%|####9 | 147/300 [47:21<45:25, 17.82s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 49%|####9 | 148/300 [47:21<47:30, 18.75s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 49%|####9 | 148/300 [47:21<47:30, 18.75s/it, lr: 1.0e-03 loss: 2.161e-02] The_Cosmic_Drift_Slider: 49%|####9 | 148/300 [47:42<47:30, 18.75s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 49%|####9 | 148/300 [47:42<47:30, 18.75s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 50%|####9 | 149/300 [47:42<48:44, 19.37s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 50%|####9 | 149/300 [47:42<48:44, 19.37s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 50%|####9 | 149/300 [48:03<48:44, 19.37s/it, lr: 1.0e-03 loss: 4.517e-02] The_Cosmic_Drift_Slider: 50%|####9 | 149/300 [48:03<48:44, 19.37s/it, lr: 1.0e-03 loss: 4.517e-02] | |
| Saving at step 150 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000150.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| Removing old save: /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000050.safetensors | |
| The_Cosmic_Drift_Slider: 50%|##### | 150/300 [48:16<48:25, 19.37s/it, lr: 1.0e-03 loss: 1.306e-02] The_Cosmic_Drift_Slider: 50%|##### | 150/300 [48:16<48:25, 19.37s/it, lr: 1.0e-03 loss: 1.306e-02] The_Cosmic_Drift_Slider: 50%|##### | 151/300 [48:16<33:28, 13.48s/it, lr: 1.0e-03 loss: 1.306e-02] The_Cosmic_Drift_Slider: 50%|##### | 151/300 [48:16<33:28, 13.48s/it, lr: 1.0e-03 loss: 1.306e-02] The_Cosmic_Drift_Slider: 50%|##### | 151/300 [48:38<33:28, 13.48s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 50%|##### | 151/300 [48:38<33:28, 13.48s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 51%|##### | 152/300 [48:38<37:52, 15.36s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 51%|##### | 152/300 [48:38<37:52, 15.36s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 51%|##### | 152/300 [48:59<37:52, 15.36s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 51%|##### | 152/300 [48:59<37:52, 15.36s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 153/300 [48:59<41:07, 16.79s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 153/300 [48:59<41:07, 16.79s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 153/300 [49:19<41:07, 16.79s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 153/300 [49:19<41:07, 16.79s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 154/300 [49:19<43:29, 17.87s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 154/300 [49:19<43:29, 17.87s/it, lr: 1.0e-03 loss: 1.648e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 154/300 [49:32<43:29, 17.87s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 51%|#####1 | 154/300 [49:32<43:29, 17.87s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 52%|#####1 | 155/300 [49:32<39:44, 16.44s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 52%|#####1 | 155/300 [49:32<39:44, 16.44s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 52%|#####1 | 155/300 [49:46<39:44, 16.44s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 52%|#####1 | 155/300 [49:46<39:44, 16.44s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 156/300 [49:46<37:02, 15.43s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 156/300 [49:46<37:02, 15.43s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 156/300 [50:07<37:02, 15.43s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 156/300 [50:07<37:02, 15.43s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 157/300 [50:07<40:30, 16.99s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 157/300 [50:07<40:30, 16.99s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 157/300 [50:27<40:30, 16.99s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 52%|#####2 | 157/300 [50:27<40:30, 16.99s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 53%|#####2 | 158/300 [50:27<42:52, 18.12s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 53%|#####2 | 158/300 [50:27<42:52, 18.12s/it, lr: 1.0e-03 loss: 5.518e-02] The_Cosmic_Drift_Slider: 53%|#####2 | 158/300 [50:48<42:52, 18.12s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 53%|#####2 | 158/300 [50:48<42:52, 18.12s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 53%|#####3 | 159/300 [50:48<44:27, 18.92s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 53%|#####3 | 159/300 [50:48<44:27, 18.92s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 53%|#####3 | 159/300 [51:10<44:27, 18.92s/it, lr: 1.0e-03 loss: 6.104e-02] The_Cosmic_Drift_Slider: 53%|#####3 | 159/300 [51:10<44:27, 18.92s/it, lr: 1.0e-03 loss: 6.104e-02] The_Cosmic_Drift_Slider: 53%|#####3 | 160/300 [51:31<44:08, 18.92s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 53%|#####3 | 160/300 [51:31<44:08, 18.92s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 54%|#####3 | 161/300 [51:31<34:49, 15.03s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 54%|#####3 | 161/300 [51:31<34:49, 15.03s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 54%|#####3 | 161/300 [51:52<34:49, 15.03s/it, lr: 1.0e-03 loss: 2.234e-02] The_Cosmic_Drift_Slider: 54%|#####3 | 161/300 [51:52<34:49, 15.03s/it, lr: 1.0e-03 loss: 2.234e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 162/300 [51:52<37:45, 16.42s/it, lr: 1.0e-03 loss: 2.234e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 162/300 [51:52<37:45, 16.42s/it, lr: 1.0e-03 loss: 2.234e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 162/300 [52:04<37:45, 16.42s/it, lr: 1.0e-03 loss: 6.934e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 162/300 [52:04<37:45, 16.42s/it, lr: 1.0e-03 loss: 6.934e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 163/300 [52:04<35:17, 15.46s/it, lr: 1.0e-03 loss: 6.934e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 163/300 [52:04<35:17, 15.46s/it, lr: 1.0e-03 loss: 6.934e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 163/300 [52:26<35:17, 15.46s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 54%|#####4 | 163/300 [52:26<35:17, 15.46s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 55%|#####4 | 164/300 [52:26<38:26, 16.96s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 55%|#####4 | 164/300 [52:26<38:26, 16.96s/it, lr: 1.0e-03 loss: 2.588e-02] The_Cosmic_Drift_Slider: 55%|#####4 | 164/300 [52:47<38:26, 16.96s/it, lr: 1.0e-03 loss: 2.759e-02] The_Cosmic_Drift_Slider: 55%|#####4 | 164/300 [52:47<38:26, 16.96s/it, lr: 1.0e-03 loss: 2.759e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 165/300 [52:47<40:34, 18.04s/it, lr: 1.0e-03 loss: 2.759e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 165/300 [52:47<40:34, 18.04s/it, lr: 1.0e-03 loss: 2.759e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 165/300 [53:08<40:34, 18.04s/it, lr: 1.0e-03 loss: 4.077e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 165/300 [53:08<40:34, 18.04s/it, lr: 1.0e-03 loss: 4.077e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 166/300 [53:08<42:03, 18.83s/it, lr: 1.0e-03 loss: 4.077e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 166/300 [53:08<42:03, 18.83s/it, lr: 1.0e-03 loss: 4.077e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 166/300 [53:20<42:03, 18.83s/it, lr: 1.0e-03 loss: 1.196e-02] The_Cosmic_Drift_Slider: 55%|#####5 | 166/300 [53:20<42:03, 18.83s/it, lr: 1.0e-03 loss: 1.196e-02] The_Cosmic_Drift_Slider: 56%|#####5 | 167/300 [53:20<37:48, 17.06s/it, lr: 1.0e-03 loss: 1.196e-02] The_Cosmic_Drift_Slider: 56%|#####5 | 167/300 [53:20<37:48, 17.06s/it, lr: 1.0e-03 loss: 1.196e-02] The_Cosmic_Drift_Slider: 56%|#####5 | 167/300 [53:34<37:48, 17.06s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 56%|#####5 | 167/300 [53:34<37:48, 17.06s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 168/300 [53:34<34:51, 15.84s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 168/300 [53:34<34:51, 15.84s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 168/300 [53:55<34:51, 15.84s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 168/300 [53:55<34:51, 15.84s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 169/300 [53:55<37:47, 17.31s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 169/300 [53:55<37:47, 17.31s/it, lr: 1.0e-03 loss: 1.611e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 169/300 [54:16<37:47, 17.31s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 56%|#####6 | 169/300 [54:16<37:47, 17.31s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 57%|#####6 | 170/300 [54:37<37:30, 17.31s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 57%|#####6 | 170/300 [54:37<37:30, 17.31s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 57%|#####6 | 171/300 [54:37<30:26, 14.16s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 57%|#####6 | 171/300 [54:37<30:26, 14.16s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 57%|#####6 | 171/300 [54:58<30:26, 14.16s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 57%|#####6 | 171/300 [54:58<30:26, 14.16s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 57%|#####7 | 172/300 [54:58<33:47, 15.84s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 57%|#####7 | 172/300 [54:58<33:47, 15.84s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 57%|#####7 | 172/300 [55:19<33:47, 15.84s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 57%|#####7 | 172/300 [55:19<33:47, 15.84s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 58%|#####7 | 173/300 [55:19<36:16, 17.14s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 58%|#####7 | 173/300 [55:19<36:16, 17.14s/it, lr: 1.0e-03 loss: 1.587e-02] The_Cosmic_Drift_Slider: 58%|#####7 | 173/300 [55:32<36:16, 17.14s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 58%|#####7 | 173/300 [55:32<36:16, 17.14s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 58%|#####8 | 174/300 [55:32<33:31, 15.96s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 58%|#####8 | 174/300 [55:32<33:31, 15.96s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 58%|#####8 | 174/300 [55:53<33:31, 15.96s/it, lr: 1.0e-03 loss: 2.490e-02] The_Cosmic_Drift_Slider: 58%|#####8 | 174/300 [55:53<33:31, 15.96s/it, lr: 1.0e-03 loss: 2.490e-02] | |
| Saving at step 175 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000175.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| Removing old save: /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000075.safetensors | |
| The_Cosmic_Drift_Slider: 58%|#####8 | 175/300 [56:15<33:15, 15.96s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 58%|#####8 | 175/300 [56:15<33:15, 15.96s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 176/300 [56:15<28:06, 13.60s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 176/300 [56:15<28:06, 13.60s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 176/300 [56:28<28:06, 13.60s/it, lr: 1.0e-03 loss: 7.422e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 176/300 [56:28<28:06, 13.60s/it, lr: 1.0e-03 loss: 7.422e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 177/300 [56:28<27:29, 13.41s/it, lr: 1.0e-03 loss: 7.422e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 177/300 [56:28<27:29, 13.41s/it, lr: 1.0e-03 loss: 7.422e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 177/300 [56:49<27:29, 13.41s/it, lr: 1.0e-03 loss: 1.526e-02] The_Cosmic_Drift_Slider: 59%|#####8 | 177/300 [56:49<27:29, 13.41s/it, lr: 1.0e-03 loss: 1.526e-02] The_Cosmic_Drift_Slider: 59%|#####9 | 178/300 [56:49<31:04, 15.28s/it, lr: 1.0e-03 loss: 1.526e-02] The_Cosmic_Drift_Slider: 59%|#####9 | 178/300 [56:49<31:04, 15.28s/it, lr: 1.0e-03 loss: 1.526e-02] The_Cosmic_Drift_Slider: 59%|#####9 | 178/300 [57:09<31:04, 15.28s/it, lr: 1.0e-03 loss: 4.956e-02] The_Cosmic_Drift_Slider: 59%|#####9 | 178/300 [57:09<31:04, 15.28s/it, lr: 1.0e-03 loss: 4.956e-02] The_Cosmic_Drift_Slider: 60%|#####9 | 179/300 [57:09<33:47, 16.75s/it, lr: 1.0e-03 loss: 4.956e-02] The_Cosmic_Drift_Slider: 60%|#####9 | 179/300 [57:09<33:47, 16.75s/it, lr: 1.0e-03 loss: 4.956e-02] The_Cosmic_Drift_Slider: 60%|#####9 | 179/300 [57:31<33:47, 16.75s/it, lr: 1.0e-03 loss: 3.857e-02] The_Cosmic_Drift_Slider: 60%|#####9 | 179/300 [57:31<33:47, 16.75s/it, lr: 1.0e-03 loss: 3.857e-02] The_Cosmic_Drift_Slider: 60%|###### | 180/300 [57:52<33:30, 16.75s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 60%|###### | 180/300 [57:52<33:30, 16.75s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 60%|###### | 181/300 [57:52<27:47, 14.02s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 60%|###### | 181/300 [57:52<27:47, 14.02s/it, lr: 1.0e-03 loss: 5.273e-02] The_Cosmic_Drift_Slider: 60%|###### | 181/300 [58:13<27:47, 14.02s/it, lr: 1.0e-03 loss: 2.893e-02] The_Cosmic_Drift_Slider: 60%|###### | 181/300 [58:13<27:47, 14.02s/it, lr: 1.0e-03 loss: 2.893e-02] The_Cosmic_Drift_Slider: 61%|###### | 182/300 [58:13<30:42, 15.62s/it, lr: 1.0e-03 loss: 2.893e-02] The_Cosmic_Drift_Slider: 61%|###### | 182/300 [58:13<30:42, 15.62s/it, lr: 1.0e-03 loss: 2.893e-02] The_Cosmic_Drift_Slider: 61%|###### | 182/300 [58:26<30:42, 15.62s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 61%|###### | 182/300 [58:26<30:42, 15.62s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 61%|######1 | 183/300 [58:26<29:01, 14.89s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 61%|######1 | 183/300 [58:26<29:01, 14.89s/it, lr: 1.0e-03 loss: 1.270e-02] The_Cosmic_Drift_Slider: 61%|######1 | 183/300 [58:47<29:01, 14.89s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 61%|######1 | 183/300 [58:47<29:01, 14.89s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 61%|######1 | 184/300 [58:47<31:54, 16.50s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 61%|######1 | 184/300 [58:47<31:54, 16.50s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 61%|######1 | 184/300 [59:00<31:54, 16.50s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 61%|######1 | 184/300 [59:00<31:54, 16.50s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 62%|######1 | 185/300 [59:00<29:39, 15.47s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 62%|######1 | 185/300 [59:00<29:39, 15.47s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 62%|######1 | 185/300 [59:21<29:39, 15.47s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 62%|######1 | 185/300 [59:21<29:39, 15.47s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 62%|######2 | 186/300 [59:21<32:15, 16.98s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 62%|######2 | 186/300 [59:21<32:15, 16.98s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 62%|######2 | 186/300 [59:42<32:15, 16.98s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 62%|######2 | 186/300 [59:42<32:15, 16.98s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 62%|######2 | 187/300 [59:42<33:57, 18.03s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 62%|######2 | 187/300 [59:42<33:57, 18.03s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 62%|######2 | 187/300 [59:55<33:57, 18.03s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 62%|######2 | 187/300 [59:55<33:57, 18.03s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 63%|######2 | 188/300 [59:55<30:53, 16.55s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 63%|######2 | 188/300 [59:55<30:53, 16.55s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 63%|######2 | 188/300 [1:00:16<30:53, 16.55s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 63%|######2 | 188/300 [1:00:16<30:53, 16.55s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 63%|######3 | 189/300 [1:00:16<32:55, 17.80s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 63%|######3 | 189/300 [1:00:16<32:55, 17.80s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 63%|######3 | 189/300 [1:00:37<32:55, 17.80s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 63%|######3 | 189/300 [1:00:37<32:55, 17.80s/it, lr: 1.0e-03 loss: 2.246e-02] The_Cosmic_Drift_Slider: 63%|######3 | 190/300 [1:00:58<32:37, 17.80s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 63%|######3 | 190/300 [1:00:58<32:37, 17.80s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 64%|######3 | 191/300 [1:00:58<26:12, 14.43s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 64%|######3 | 191/300 [1:00:58<26:12, 14.43s/it, lr: 1.0e-03 loss: 1.929e-02] The_Cosmic_Drift_Slider: 64%|######3 | 191/300 [1:01:19<26:12, 14.43s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 64%|######3 | 191/300 [1:01:19<26:12, 14.43s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 64%|######4 | 192/300 [1:01:19<28:52, 16.04s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 64%|######4 | 192/300 [1:01:19<28:52, 16.04s/it, lr: 1.0e-03 loss: 3.467e-02] The_Cosmic_Drift_Slider: 64%|######4 | 192/300 [1:01:40<28:52, 16.04s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 64%|######4 | 192/300 [1:01:40<28:52, 16.04s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 64%|######4 | 193/300 [1:01:40<30:51, 17.30s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 64%|######4 | 193/300 [1:01:40<30:51, 17.30s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 64%|######4 | 193/300 [1:01:53<30:51, 17.30s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 64%|######4 | 193/300 [1:01:53<30:51, 17.30s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 65%|######4 | 194/300 [1:01:53<28:24, 16.08s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 65%|######4 | 194/300 [1:01:53<28:24, 16.08s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 65%|######4 | 194/300 [1:02:14<28:24, 16.08s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 65%|######4 | 194/300 [1:02:14<28:24, 16.08s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 65%|######5 | 195/300 [1:02:14<30:27, 17.41s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 65%|######5 | 195/300 [1:02:14<30:27, 17.41s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 65%|######5 | 195/300 [1:02:36<30:27, 17.41s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 65%|######5 | 195/300 [1:02:36<30:27, 17.41s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 65%|######5 | 196/300 [1:02:36<31:56, 18.42s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 65%|######5 | 196/300 [1:02:36<31:56, 18.42s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 65%|######5 | 196/300 [1:02:57<31:56, 18.42s/it, lr: 1.0e-03 loss: 1.868e-02] The_Cosmic_Drift_Slider: 65%|######5 | 196/300 [1:02:57<31:56, 18.42s/it, lr: 1.0e-03 loss: 1.868e-02] The_Cosmic_Drift_Slider: 66%|######5 | 197/300 [1:02:57<32:49, 19.12s/it, lr: 1.0e-03 loss: 1.868e-02] The_Cosmic_Drift_Slider: 66%|######5 | 197/300 [1:02:57<32:49, 19.12s/it, lr: 1.0e-03 loss: 1.868e-02] The_Cosmic_Drift_Slider: 66%|######5 | 197/300 [1:03:09<32:49, 19.12s/it, lr: 1.0e-03 loss: 1.575e-02] The_Cosmic_Drift_Slider: 66%|######5 | 197/300 [1:03:09<32:49, 19.12s/it, lr: 1.0e-03 loss: 1.575e-02] The_Cosmic_Drift_Slider: 66%|######6 | 198/300 [1:03:09<29:19, 17.25s/it, lr: 1.0e-03 loss: 1.575e-02] The_Cosmic_Drift_Slider: 66%|######6 | 198/300 [1:03:09<29:19, 17.25s/it, lr: 1.0e-03 loss: 1.575e-02] The_Cosmic_Drift_Slider: 66%|######6 | 198/300 [1:03:30<29:19, 17.25s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 66%|######6 | 198/300 [1:03:30<29:19, 17.25s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 66%|######6 | 199/300 [1:03:30<30:49, 18.31s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 66%|######6 | 199/300 [1:03:30<30:49, 18.31s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 66%|######6 | 199/300 [1:03:44<30:49, 18.31s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 66%|######6 | 199/300 [1:03:44<30:49, 18.31s/it, lr: 1.0e-03 loss: 1.257e-02] | |
| Saving at step 200 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000200.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| Removing old save: /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000100.safetensors | |
| The_Cosmic_Drift_Slider: 67%|######6 | 200/300 [1:04:05<30:31, 18.31s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 67%|######6 | 200/300 [1:04:05<30:31, 18.31s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 67%|######7 | 201/300 [1:04:05<24:23, 14.78s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 67%|######7 | 201/300 [1:04:05<24:23, 14.78s/it, lr: 1.0e-03 loss: 1.855e-02] The_Cosmic_Drift_Slider: 67%|######7 | 201/300 [1:04:26<24:23, 14.78s/it, lr: 1.0e-03 loss: 5.322e-02] The_Cosmic_Drift_Slider: 67%|######7 | 201/300 [1:04:26<24:23, 14.78s/it, lr: 1.0e-03 loss: 5.322e-02] The_Cosmic_Drift_Slider: 67%|######7 | 202/300 [1:04:26<26:33, 16.26s/it, lr: 1.0e-03 loss: 5.322e-02] The_Cosmic_Drift_Slider: 67%|######7 | 202/300 [1:04:26<26:33, 16.26s/it, lr: 1.0e-03 loss: 5.322e-02] The_Cosmic_Drift_Slider: 67%|######7 | 202/300 [1:04:46<26:33, 16.26s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 67%|######7 | 202/300 [1:04:46<26:33, 16.26s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 68%|######7 | 203/300 [1:04:46<28:12, 17.45s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 68%|######7 | 203/300 [1:04:46<28:12, 17.45s/it, lr: 1.0e-03 loss: 4.980e-02] The_Cosmic_Drift_Slider: 68%|######7 | 203/300 [1:05:08<28:12, 17.45s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 68%|######7 | 203/300 [1:05:08<28:12, 17.45s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 68%|######8 | 204/300 [1:05:08<29:26, 18.41s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 68%|######8 | 204/300 [1:05:08<29:26, 18.41s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 68%|######8 | 204/300 [1:05:29<29:26, 18.41s/it, lr: 1.0e-03 loss: 2.576e-02] The_Cosmic_Drift_Slider: 68%|######8 | 204/300 [1:05:29<29:26, 18.41s/it, lr: 1.0e-03 loss: 2.576e-02] The_Cosmic_Drift_Slider: 68%|######8 | 205/300 [1:05:29<30:12, 19.08s/it, lr: 1.0e-03 loss: 2.576e-02] The_Cosmic_Drift_Slider: 68%|######8 | 205/300 [1:05:29<30:12, 19.08s/it, lr: 1.0e-03 loss: 2.576e-02] The_Cosmic_Drift_Slider: 68%|######8 | 205/300 [1:05:42<30:12, 19.08s/it, lr: 1.0e-03 loss: 2.686e-02] The_Cosmic_Drift_Slider: 68%|######8 | 205/300 [1:05:42<30:12, 19.08s/it, lr: 1.0e-03 loss: 2.686e-02] The_Cosmic_Drift_Slider: 69%|######8 | 206/300 [1:05:42<27:03, 17.27s/it, lr: 1.0e-03 loss: 2.686e-02] The_Cosmic_Drift_Slider: 69%|######8 | 206/300 [1:05:42<27:03, 17.27s/it, lr: 1.0e-03 loss: 2.686e-02] The_Cosmic_Drift_Slider: 69%|######8 | 206/300 [1:06:03<27:03, 17.27s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 69%|######8 | 206/300 [1:06:03<27:03, 17.27s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 69%|######9 | 207/300 [1:06:03<28:21, 18.30s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 69%|######9 | 207/300 [1:06:03<28:21, 18.30s/it, lr: 1.0e-03 loss: 2.124e-02] The_Cosmic_Drift_Slider: 69%|######9 | 207/300 [1:06:24<28:21, 18.30s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 69%|######9 | 207/300 [1:06:24<28:21, 18.30s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 69%|######9 | 208/300 [1:06:24<29:16, 19.10s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 69%|######9 | 208/300 [1:06:24<29:16, 19.10s/it, lr: 1.0e-03 loss: 1.501e-02] The_Cosmic_Drift_Slider: 69%|######9 | 208/300 [1:06:45<29:16, 19.10s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 69%|######9 | 208/300 [1:06:45<29:16, 19.10s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 70%|######9 | 209/300 [1:06:45<29:44, 19.61s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 70%|######9 | 209/300 [1:06:45<29:44, 19.61s/it, lr: 1.0e-03 loss: 2.222e-02] The_Cosmic_Drift_Slider: 70%|######9 | 209/300 [1:06:58<29:44, 19.61s/it, lr: 1.0e-03 loss: 1.306e-02] The_Cosmic_Drift_Slider: 70%|######9 | 209/300 [1:06:58<29:44, 19.61s/it, lr: 1.0e-03 loss: 1.306e-02] The_Cosmic_Drift_Slider: 70%|####### | 210/300 [1:07:19<29:24, 19.61s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 70%|####### | 210/300 [1:07:19<29:24, 19.61s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 70%|####### | 211/300 [1:07:19<22:51, 15.41s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 70%|####### | 211/300 [1:07:19<22:51, 15.41s/it, lr: 1.0e-03 loss: 1.917e-02] The_Cosmic_Drift_Slider: 70%|####### | 211/300 [1:07:40<22:51, 15.41s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 70%|####### | 211/300 [1:07:40<22:51, 15.41s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 71%|####### | 212/300 [1:07:40<24:31, 16.72s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 71%|####### | 212/300 [1:07:40<24:31, 16.72s/it, lr: 1.0e-03 loss: 3.760e-02] The_Cosmic_Drift_Slider: 71%|####### | 212/300 [1:08:01<24:31, 16.72s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 71%|####### | 212/300 [1:08:01<24:31, 16.72s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 213/300 [1:08:01<25:49, 17.82s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 213/300 [1:08:01<25:49, 17.82s/it, lr: 1.0e-03 loss: 2.014e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 213/300 [1:08:22<25:49, 17.82s/it, lr: 1.0e-03 loss: 2.271e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 213/300 [1:08:22<25:49, 17.82s/it, lr: 1.0e-03 loss: 2.271e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 214/300 [1:08:22<26:42, 18.63s/it, lr: 1.0e-03 loss: 2.271e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 214/300 [1:08:22<26:42, 18.63s/it, lr: 1.0e-03 loss: 2.271e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 214/300 [1:08:35<26:42, 18.63s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 71%|#######1 | 214/300 [1:08:35<26:42, 18.63s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 72%|#######1 | 215/300 [1:08:35<24:03, 16.98s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 72%|#######1 | 215/300 [1:08:35<24:03, 16.98s/it, lr: 1.0e-03 loss: 1.416e-02] The_Cosmic_Drift_Slider: 72%|#######1 | 215/300 [1:08:49<24:03, 16.98s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 72%|#######1 | 215/300 [1:08:49<24:03, 16.98s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 72%|#######2 | 216/300 [1:08:49<22:09, 15.83s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 72%|#######2 | 216/300 [1:08:49<22:09, 15.83s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 72%|#######2 | 216/300 [1:09:09<22:09, 15.83s/it, lr: 1.0e-03 loss: 1.064e-01] The_Cosmic_Drift_Slider: 72%|#######2 | 216/300 [1:09:09<22:09, 15.83s/it, lr: 1.0e-03 loss: 1.064e-01] The_Cosmic_Drift_Slider: 72%|#######2 | 217/300 [1:09:09<23:53, 17.27s/it, lr: 1.0e-03 loss: 1.064e-01] The_Cosmic_Drift_Slider: 72%|#######2 | 217/300 [1:09:09<23:53, 17.27s/it, lr: 1.0e-03 loss: 1.064e-01] The_Cosmic_Drift_Slider: 72%|#######2 | 217/300 [1:09:30<23:53, 17.27s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 72%|#######2 | 217/300 [1:09:30<23:53, 17.27s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 73%|#######2 | 218/300 [1:09:30<25:01, 18.31s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 73%|#######2 | 218/300 [1:09:30<25:01, 18.31s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 73%|#######2 | 218/300 [1:09:51<25:01, 18.31s/it, lr: 1.0e-03 loss: 1.556e-02] The_Cosmic_Drift_Slider: 73%|#######2 | 218/300 [1:09:51<25:01, 18.31s/it, lr: 1.0e-03 loss: 1.556e-02] The_Cosmic_Drift_Slider: 73%|#######3 | 219/300 [1:09:51<25:43, 19.05s/it, lr: 1.0e-03 loss: 1.556e-02] The_Cosmic_Drift_Slider: 73%|#######3 | 219/300 [1:09:51<25:43, 19.05s/it, lr: 1.0e-03 loss: 1.556e-02] The_Cosmic_Drift_Slider: 73%|#######3 | 219/300 [1:10:13<25:43, 19.05s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 73%|#######3 | 219/300 [1:10:13<25:43, 19.05s/it, lr: 1.0e-03 loss: 2.039e-02] The_Cosmic_Drift_Slider: 73%|#######3 | 220/300 [1:10:34<25:24, 19.05s/it, lr: 1.0e-03 loss: 1.514e-02] The_Cosmic_Drift_Slider: 73%|#######3 | 220/300 [1:10:34<25:24, 19.05s/it, lr: 1.0e-03 loss: 1.514e-02] The_Cosmic_Drift_Slider: 74%|#######3 | 221/300 [1:10:34<19:52, 15.10s/it, lr: 1.0e-03 loss: 1.514e-02] The_Cosmic_Drift_Slider: 74%|#######3 | 221/300 [1:10:34<19:52, 15.10s/it, lr: 1.0e-03 loss: 1.514e-02] The_Cosmic_Drift_Slider: 74%|#######3 | 221/300 [1:10:46<19:52, 15.10s/it, lr: 1.0e-03 loss: 1.404e-02] The_Cosmic_Drift_Slider: 74%|#######3 | 221/300 [1:10:46<19:52, 15.10s/it, lr: 1.0e-03 loss: 1.404e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 222/300 [1:10:46<18:52, 14.51s/it, lr: 1.0e-03 loss: 1.404e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 222/300 [1:10:46<18:52, 14.51s/it, lr: 1.0e-03 loss: 1.404e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 222/300 [1:11:07<18:52, 14.51s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 222/300 [1:11:07<18:52, 14.51s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 223/300 [1:11:07<20:43, 16.15s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 223/300 [1:11:07<20:43, 16.15s/it, lr: 1.0e-03 loss: 2.148e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 223/300 [1:11:29<20:43, 16.15s/it, lr: 1.0e-03 loss: 2.881e-02] The_Cosmic_Drift_Slider: 74%|#######4 | 223/300 [1:11:29<20:43, 16.15s/it, lr: 1.0e-03 loss: 2.881e-02] The_Cosmic_Drift_Slider: 75%|#######4 | 224/300 [1:11:29<22:07, 17.46s/it, lr: 1.0e-03 loss: 2.881e-02] The_Cosmic_Drift_Slider: 75%|#######4 | 224/300 [1:11:29<22:07, 17.46s/it, lr: 1.0e-03 loss: 2.881e-02] The_Cosmic_Drift_Slider: 75%|#######4 | 224/300 [1:11:50<22:07, 17.46s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 75%|#######4 | 224/300 [1:11:50<22:07, 17.46s/it, lr: 1.0e-03 loss: 4.590e-02] | |
| Saving at step 225 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000225.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| Removing old save: /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000125.safetensors | |
| The_Cosmic_Drift_Slider: 75%|#######5 | 225/300 [1:12:11<21:49, 17.46s/it, lr: 1.0e-03 loss: 3.223e-02] The_Cosmic_Drift_Slider: 75%|#######5 | 225/300 [1:12:11<21:49, 17.46s/it, lr: 1.0e-03 loss: 3.223e-02] The_Cosmic_Drift_Slider: 75%|#######5 | 226/300 [1:12:11<17:54, 14.53s/it, lr: 1.0e-03 loss: 3.223e-02] The_Cosmic_Drift_Slider: 75%|#######5 | 226/300 [1:12:11<17:54, 14.53s/it, lr: 1.0e-03 loss: 3.223e-02] The_Cosmic_Drift_Slider: 75%|#######5 | 226/300 [1:12:24<17:54, 14.53s/it, lr: 1.0e-03 loss: 1.184e-02] The_Cosmic_Drift_Slider: 75%|#######5 | 226/300 [1:12:24<17:54, 14.53s/it, lr: 1.0e-03 loss: 1.184e-02] The_Cosmic_Drift_Slider: 76%|#######5 | 227/300 [1:12:24<17:09, 14.10s/it, lr: 1.0e-03 loss: 1.184e-02] The_Cosmic_Drift_Slider: 76%|#######5 | 227/300 [1:12:24<17:09, 14.10s/it, lr: 1.0e-03 loss: 1.184e-02] The_Cosmic_Drift_Slider: 76%|#######5 | 227/300 [1:12:46<17:09, 14.10s/it, lr: 1.0e-03 loss: 2.441e-02] The_Cosmic_Drift_Slider: 76%|#######5 | 227/300 [1:12:46<17:09, 14.10s/it, lr: 1.0e-03 loss: 2.441e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 228/300 [1:12:46<19:00, 15.84s/it, lr: 1.0e-03 loss: 2.441e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 228/300 [1:12:46<19:00, 15.84s/it, lr: 1.0e-03 loss: 2.441e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 228/300 [1:13:06<19:00, 15.84s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 228/300 [1:13:06<19:00, 15.84s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 229/300 [1:13:06<20:18, 17.16s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 229/300 [1:13:06<20:18, 17.16s/it, lr: 1.0e-03 loss: 2.026e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 229/300 [1:13:19<20:18, 17.16s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 76%|#######6 | 229/300 [1:13:19<20:18, 17.16s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 77%|#######6 | 230/300 [1:13:40<20:01, 17.16s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 77%|#######6 | 230/300 [1:13:40<20:01, 17.16s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 231/300 [1:13:40<16:22, 14.24s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 231/300 [1:13:40<16:22, 14.24s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 231/300 [1:14:02<16:22, 14.24s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 231/300 [1:14:02<16:22, 14.24s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 232/300 [1:14:02<17:57, 15.85s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 232/300 [1:14:02<17:57, 15.85s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 232/300 [1:14:23<17:57, 15.85s/it, lr: 1.0e-03 loss: 2.783e-02] The_Cosmic_Drift_Slider: 77%|#######7 | 232/300 [1:14:23<17:57, 15.85s/it, lr: 1.0e-03 loss: 2.783e-02] The_Cosmic_Drift_Slider: 78%|#######7 | 233/300 [1:14:23<19:05, 17.10s/it, lr: 1.0e-03 loss: 2.783e-02] The_Cosmic_Drift_Slider: 78%|#######7 | 233/300 [1:14:23<19:05, 17.10s/it, lr: 1.0e-03 loss: 2.783e-02] The_Cosmic_Drift_Slider: 78%|#######7 | 233/300 [1:14:43<19:05, 17.10s/it, lr: 1.0e-03 loss: 3.784e-02] The_Cosmic_Drift_Slider: 78%|#######7 | 233/300 [1:14:43<19:05, 17.10s/it, lr: 1.0e-03 loss: 3.784e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 234/300 [1:14:43<19:53, 18.09s/it, lr: 1.0e-03 loss: 3.784e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 234/300 [1:14:43<19:53, 18.09s/it, lr: 1.0e-03 loss: 3.784e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 234/300 [1:14:56<19:53, 18.09s/it, lr: 1.0e-03 loss: 4.541e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 234/300 [1:14:56<19:53, 18.09s/it, lr: 1.0e-03 loss: 4.541e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 235/300 [1:14:56<18:00, 16.63s/it, lr: 1.0e-03 loss: 4.541e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 235/300 [1:14:56<18:00, 16.63s/it, lr: 1.0e-03 loss: 4.541e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 235/300 [1:15:18<18:00, 16.63s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 78%|#######8 | 235/300 [1:15:18<18:00, 16.63s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 79%|#######8 | 236/300 [1:15:18<19:03, 17.86s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 79%|#######8 | 236/300 [1:15:18<19:03, 17.86s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 79%|#######8 | 236/300 [1:15:31<19:03, 17.86s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 79%|#######8 | 236/300 [1:15:31<19:03, 17.86s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 237/300 [1:15:31<17:09, 16.34s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 237/300 [1:15:31<17:09, 16.34s/it, lr: 1.0e-03 loss: 1.257e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 237/300 [1:15:51<17:09, 16.34s/it, lr: 1.0e-03 loss: 2.002e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 237/300 [1:15:51<17:09, 16.34s/it, lr: 1.0e-03 loss: 2.002e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 238/300 [1:15:51<18:14, 17.65s/it, lr: 1.0e-03 loss: 2.002e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 238/300 [1:15:51<18:14, 17.65s/it, lr: 1.0e-03 loss: 2.002e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 238/300 [1:16:12<18:14, 17.65s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 79%|#######9 | 238/300 [1:16:12<18:14, 17.65s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 80%|#######9 | 239/300 [1:16:12<18:53, 18.58s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 80%|#######9 | 239/300 [1:16:12<18:53, 18.58s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 80%|#######9 | 239/300 [1:16:34<18:53, 18.58s/it, lr: 1.0e-03 loss: 2.698e-02] The_Cosmic_Drift_Slider: 80%|#######9 | 239/300 [1:16:34<18:53, 18.58s/it, lr: 1.0e-03 loss: 2.698e-02] The_Cosmic_Drift_Slider: 80%|######## | 240/300 [1:16:55<18:34, 18.58s/it, lr: 1.0e-03 loss: 2.563e-02] The_Cosmic_Drift_Slider: 80%|######## | 240/300 [1:16:55<18:34, 18.58s/it, lr: 1.0e-03 loss: 2.563e-02] The_Cosmic_Drift_Slider: 80%|######## | 241/300 [1:16:55<14:36, 14.86s/it, lr: 1.0e-03 loss: 2.563e-02] The_Cosmic_Drift_Slider: 80%|######## | 241/300 [1:16:55<14:36, 14.86s/it, lr: 1.0e-03 loss: 2.563e-02] The_Cosmic_Drift_Slider: 80%|######## | 241/300 [1:17:16<14:36, 14.86s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 80%|######## | 241/300 [1:17:16<14:36, 14.86s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 81%|######## | 242/300 [1:17:16<15:46, 16.33s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 81%|######## | 242/300 [1:17:16<15:46, 16.33s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 81%|######## | 242/300 [1:17:28<15:46, 16.33s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 81%|######## | 242/300 [1:17:28<15:46, 16.33s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 81%|########1 | 243/300 [1:17:28<14:37, 15.39s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 81%|########1 | 243/300 [1:17:28<14:37, 15.39s/it, lr: 1.0e-03 loss: 1.367e-02] The_Cosmic_Drift_Slider: 81%|########1 | 243/300 [1:17:50<14:37, 15.39s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 81%|########1 | 243/300 [1:17:50<14:37, 15.39s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 81%|########1 | 244/300 [1:17:50<15:46, 16.90s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 81%|########1 | 244/300 [1:17:50<15:46, 16.90s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 81%|########1 | 244/300 [1:18:11<15:46, 16.90s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 81%|########1 | 244/300 [1:18:11<15:46, 16.90s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 82%|########1 | 245/300 [1:18:11<16:29, 17.98s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 82%|########1 | 245/300 [1:18:11<16:29, 17.98s/it, lr: 1.0e-03 loss: 2.075e-02] The_Cosmic_Drift_Slider: 82%|########1 | 245/300 [1:18:32<16:29, 17.98s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 82%|########1 | 245/300 [1:18:32<16:29, 17.98s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 82%|########2 | 246/300 [1:18:32<16:55, 18.80s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 82%|########2 | 246/300 [1:18:32<16:55, 18.80s/it, lr: 1.0e-03 loss: 1.373e-02] The_Cosmic_Drift_Slider: 82%|########2 | 246/300 [1:18:44<16:55, 18.80s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 82%|########2 | 246/300 [1:18:44<16:55, 18.80s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 82%|########2 | 247/300 [1:18:44<15:03, 17.04s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 82%|########2 | 247/300 [1:18:44<15:03, 17.04s/it, lr: 1.0e-03 loss: 1.538e-02] The_Cosmic_Drift_Slider: 82%|########2 | 247/300 [1:19:06<15:03, 17.04s/it, lr: 1.0e-03 loss: 1.471e-02] The_Cosmic_Drift_Slider: 82%|########2 | 247/300 [1:19:06<15:03, 17.04s/it, lr: 1.0e-03 loss: 1.471e-02] The_Cosmic_Drift_Slider: 83%|########2 | 248/300 [1:19:06<15:46, 18.19s/it, lr: 1.0e-03 loss: 1.471e-02] The_Cosmic_Drift_Slider: 83%|########2 | 248/300 [1:19:06<15:46, 18.19s/it, lr: 1.0e-03 loss: 1.471e-02] The_Cosmic_Drift_Slider: 83%|########2 | 248/300 [1:19:27<15:46, 18.19s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 83%|########2 | 248/300 [1:19:27<15:46, 18.19s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 83%|########2 | 249/300 [1:19:27<16:07, 18.96s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 83%|########2 | 249/300 [1:19:27<16:07, 18.96s/it, lr: 1.0e-03 loss: 2.051e-02] The_Cosmic_Drift_Slider: 83%|########2 | 249/300 [1:19:48<16:07, 18.96s/it, lr: 1.0e-03 loss: 6.934e-02] The_Cosmic_Drift_Slider: 83%|########2 | 249/300 [1:19:48<16:07, 18.96s/it, lr: 1.0e-03 loss: 6.934e-02] | |
| Saving at step 250 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000250.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| Removing old save: /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000150.safetensors | |
| Generating Images: 0%| | 0/7 [00:00<?, ?it/s] Generating Images: 0%| | 0/7 [00:00<?, ?it/s][A[A | |
| Generating Images: 14%|#4 | 1/7 [01:36<09:39, 96.50s/it] Generating Images: 14%|#4 | 1/7 [01:36<09:39, 96.50s/it][A[A | |
| Generating Images: 29%|##8 | 2/7 [03:14<08:06, 97.30s/it] Generating Images: 29%|##8 | 2/7 [03:14<08:06, 97.30s/it][A[A | |
| Generating Images: 43%|####2 | 3/7 [04:52<06:31, 97.79s/it] Generating Images: 43%|####2 | 3/7 [04:52<06:31, 97.79s/it][A[A | |
| Generating Images: 57%|#####7 | 4/7 [06:17<04:38, 92.71s/it] Generating Images: 57%|#####7 | 4/7 [06:17<04:38, 92.71s/it][A[A | |
| Generating Images: 71%|#######1 | 5/7 [07:55<03:09, 94.59s/it] Generating Images: 71%|#######1 | 5/7 [07:55<03:09, 94.59s/it][A[A | |
| Generating Images: 86%|########5 | 6/7 [09:32<01:35, 95.40s/it] Generating Images: 86%|########5 | 6/7 [09:32<01:35, 95.40s/it][A[A | |
| Generating Images: 100%|##########| 7/7 [11:10<00:00, 96.37s/it] Generating Images: 100%|##########| 7/7 [11:10<00:00, 96.37s/it][A[A | |
| [A[A The_Cosmic_Drift_Slider: 83%|########3 | 250/300 [1:20:01<15:48, 18.96s/it, lr: 1.0e-03 loss: 1.459e-02] The_Cosmic_Drift_Slider: 83%|########3 | 250/300 [1:20:01<15:48, 18.96s/it, lr: 1.0e-03 loss: 1.459e-02] The_Cosmic_Drift_Slider: 84%|########3 | 251/300 [1:20:01<10:49, 13.26s/it, lr: 1.0e-03 loss: 1.459e-02] The_Cosmic_Drift_Slider: 84%|########3 | 251/300 [1:20:01<10:49, 13.26s/it, lr: 1.0e-03 loss: 1.459e-02] The_Cosmic_Drift_Slider: 84%|########3 | 251/300 [1:20:23<10:49, 13.26s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 84%|########3 | 251/300 [1:20:23<10:49, 13.26s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 84%|########4 | 252/300 [1:20:23<12:08, 15.17s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 84%|########4 | 252/300 [1:20:23<12:08, 15.17s/it, lr: 1.0e-03 loss: 1.733e-02] The_Cosmic_Drift_Slider: 84%|########4 | 252/300 [1:20:44<12:08, 15.17s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 84%|########4 | 252/300 [1:20:44<12:08, 15.17s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 84%|########4 | 253/300 [1:20:44<13:02, 16.64s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 84%|########4 | 253/300 [1:20:44<13:02, 16.64s/it, lr: 1.0e-03 loss: 2.173e-02] The_Cosmic_Drift_Slider: 84%|########4 | 253/300 [1:21:05<13:02, 16.64s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 84%|########4 | 253/300 [1:21:05<13:02, 16.64s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 85%|########4 | 254/300 [1:21:05<13:37, 17.77s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 85%|########4 | 254/300 [1:21:05<13:37, 17.77s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 85%|########4 | 254/300 [1:21:17<13:37, 17.77s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 85%|########4 | 254/300 [1:21:17<13:37, 17.77s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 85%|########5 | 255/300 [1:21:17<12:16, 16.36s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 85%|########5 | 255/300 [1:21:17<12:16, 16.36s/it, lr: 1.0e-03 loss: 1.831e-02] The_Cosmic_Drift_Slider: 85%|########5 | 255/300 [1:21:39<12:16, 16.36s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 85%|########5 | 255/300 [1:21:39<12:16, 16.36s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 85%|########5 | 256/300 [1:21:39<12:58, 17.69s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 85%|########5 | 256/300 [1:21:39<12:58, 17.69s/it, lr: 1.0e-03 loss: 1.978e-02] The_Cosmic_Drift_Slider: 85%|########5 | 256/300 [1:21:52<12:58, 17.69s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 85%|########5 | 256/300 [1:21:52<12:58, 17.69s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 86%|########5 | 257/300 [1:21:52<11:38, 16.25s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 86%|########5 | 257/300 [1:21:52<11:38, 16.25s/it, lr: 1.0e-03 loss: 1.599e-02] The_Cosmic_Drift_Slider: 86%|########5 | 257/300 [1:22:13<11:38, 16.25s/it, lr: 1.0e-03 loss: 1.440e-02] The_Cosmic_Drift_Slider: 86%|########5 | 257/300 [1:22:13<11:38, 16.25s/it, lr: 1.0e-03 loss: 1.440e-02] The_Cosmic_Drift_Slider: 86%|########6 | 258/300 [1:22:13<12:18, 17.59s/it, lr: 1.0e-03 loss: 1.440e-02] The_Cosmic_Drift_Slider: 86%|########6 | 258/300 [1:22:13<12:18, 17.59s/it, lr: 1.0e-03 loss: 1.440e-02] The_Cosmic_Drift_Slider: 86%|########6 | 258/300 [1:22:34<12:18, 17.59s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 86%|########6 | 258/300 [1:22:34<12:18, 17.59s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 86%|########6 | 259/300 [1:22:34<12:40, 18.54s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 86%|########6 | 259/300 [1:22:34<12:40, 18.54s/it, lr: 1.0e-03 loss: 1.782e-02] The_Cosmic_Drift_Slider: 86%|########6 | 259/300 [1:22:55<12:40, 18.54s/it, lr: 1.0e-03 loss: 1.660e-02] The_Cosmic_Drift_Slider: 86%|########6 | 259/300 [1:22:55<12:40, 18.54s/it, lr: 1.0e-03 loss: 1.660e-02] The_Cosmic_Drift_Slider: 87%|########6 | 260/300 [1:23:16<12:21, 18.54s/it, lr: 1.0e-03 loss: 6.006e-02] The_Cosmic_Drift_Slider: 87%|########6 | 260/300 [1:23:16<12:21, 18.54s/it, lr: 1.0e-03 loss: 6.006e-02] The_Cosmic_Drift_Slider: 87%|########7 | 261/300 [1:23:16<09:38, 14.83s/it, lr: 1.0e-03 loss: 6.006e-02] The_Cosmic_Drift_Slider: 87%|########7 | 261/300 [1:23:16<09:38, 14.83s/it, lr: 1.0e-03 loss: 6.006e-02] The_Cosmic_Drift_Slider: 87%|########7 | 261/300 [1:23:29<09:38, 14.83s/it, lr: 1.0e-03 loss: 8.972e-03] The_Cosmic_Drift_Slider: 87%|########7 | 261/300 [1:23:29<09:38, 14.83s/it, lr: 1.0e-03 loss: 8.972e-03] The_Cosmic_Drift_Slider: 87%|########7 | 262/300 [1:23:29<09:03, 14.29s/it, lr: 1.0e-03 loss: 8.972e-03] The_Cosmic_Drift_Slider: 87%|########7 | 262/300 [1:23:29<09:03, 14.29s/it, lr: 1.0e-03 loss: 8.972e-03] The_Cosmic_Drift_Slider: 87%|########7 | 262/300 [1:23:50<09:03, 14.29s/it, lr: 1.0e-03 loss: 2.063e-02] The_Cosmic_Drift_Slider: 87%|########7 | 262/300 [1:23:50<09:03, 14.29s/it, lr: 1.0e-03 loss: 2.063e-02] The_Cosmic_Drift_Slider: 88%|########7 | 263/300 [1:23:50<09:51, 16.00s/it, lr: 1.0e-03 loss: 2.063e-02] The_Cosmic_Drift_Slider: 88%|########7 | 263/300 [1:23:50<09:51, 16.00s/it, lr: 1.0e-03 loss: 2.063e-02] The_Cosmic_Drift_Slider: 88%|########7 | 263/300 [1:24:12<09:51, 16.00s/it, lr: 1.0e-03 loss: 1.294e-02] The_Cosmic_Drift_Slider: 88%|########7 | 263/300 [1:24:12<09:51, 16.00s/it, lr: 1.0e-03 loss: 1.294e-02] The_Cosmic_Drift_Slider: 88%|########8 | 264/300 [1:24:12<10:25, 17.36s/it, lr: 1.0e-03 loss: 1.294e-02] The_Cosmic_Drift_Slider: 88%|########8 | 264/300 [1:24:12<10:25, 17.36s/it, lr: 1.0e-03 loss: 1.294e-02] The_Cosmic_Drift_Slider: 88%|########8 | 264/300 [1:24:32<10:25, 17.36s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 88%|########8 | 264/300 [1:24:32<10:25, 17.36s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 88%|########8 | 265/300 [1:24:32<10:41, 18.34s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 88%|########8 | 265/300 [1:24:32<10:41, 18.34s/it, lr: 1.0e-03 loss: 1.880e-02] The_Cosmic_Drift_Slider: 88%|########8 | 265/300 [1:24:45<10:41, 18.34s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 88%|########8 | 265/300 [1:24:45<10:41, 18.34s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 89%|########8 | 266/300 [1:24:45<09:29, 16.76s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 89%|########8 | 266/300 [1:24:45<09:29, 16.76s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 89%|########8 | 266/300 [1:25:06<09:29, 16.76s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 89%|########8 | 266/300 [1:25:06<09:29, 16.76s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 89%|########9 | 267/300 [1:25:06<09:52, 17.95s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 89%|########9 | 267/300 [1:25:06<09:52, 17.95s/it, lr: 1.0e-03 loss: 3.198e-02] The_Cosmic_Drift_Slider: 89%|########9 | 267/300 [1:25:28<09:52, 17.95s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 89%|########9 | 267/300 [1:25:28<09:52, 17.95s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 89%|########9 | 268/300 [1:25:28<10:03, 18.85s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 89%|########9 | 268/300 [1:25:28<10:03, 18.85s/it, lr: 1.0e-03 loss: 5.029e-02] The_Cosmic_Drift_Slider: 89%|########9 | 268/300 [1:25:49<10:03, 18.85s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 89%|########9 | 268/300 [1:25:49<10:03, 18.85s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 90%|########9 | 269/300 [1:25:49<10:02, 19.43s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 90%|########9 | 269/300 [1:25:49<10:02, 19.43s/it, lr: 1.0e-03 loss: 4.590e-02] The_Cosmic_Drift_Slider: 90%|########9 | 269/300 [1:26:10<10:02, 19.43s/it, lr: 1.0e-03 loss: 1.819e-02] The_Cosmic_Drift_Slider: 90%|########9 | 269/300 [1:26:10<10:02, 19.43s/it, lr: 1.0e-03 loss: 1.819e-02] The_Cosmic_Drift_Slider: 90%|######### | 270/300 [1:26:22<09:42, 19.43s/it, lr: 1.0e-03 loss: 3.809e-02] The_Cosmic_Drift_Slider: 90%|######### | 270/300 [1:26:22<09:42, 19.43s/it, lr: 1.0e-03 loss: 3.809e-02] The_Cosmic_Drift_Slider: 90%|######### | 271/300 [1:26:22<06:30, 13.45s/it, lr: 1.0e-03 loss: 3.809e-02] The_Cosmic_Drift_Slider: 90%|######### | 271/300 [1:26:22<06:30, 13.45s/it, lr: 1.0e-03 loss: 3.809e-02] The_Cosmic_Drift_Slider: 90%|######### | 271/300 [1:26:44<06:30, 13.45s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 90%|######### | 271/300 [1:26:44<06:30, 13.45s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 91%|######### | 272/300 [1:26:44<07:08, 15.31s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 91%|######### | 272/300 [1:26:44<07:08, 15.31s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 91%|######### | 272/300 [1:27:05<07:08, 15.31s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 91%|######### | 272/300 [1:27:05<07:08, 15.31s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 91%|#########1| 273/300 [1:27:05<07:32, 16.75s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 91%|#########1| 273/300 [1:27:05<07:32, 16.75s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 91%|#########1| 273/300 [1:27:26<07:32, 16.75s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 91%|#########1| 273/300 [1:27:26<07:32, 16.75s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 91%|#########1| 274/300 [1:27:26<07:44, 17.86s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 91%|#########1| 274/300 [1:27:26<07:44, 17.86s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 91%|#########1| 274/300 [1:27:38<07:44, 17.86s/it, lr: 1.0e-03 loss: 9.827e-03] The_Cosmic_Drift_Slider: 91%|#########1| 274/300 [1:27:38<07:44, 17.86s/it, lr: 1.0e-03 loss: 9.827e-03] | |
| Saving at step 275 | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000275.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |
| Removing old save: /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider_000000175.safetensors | |
| The_Cosmic_Drift_Slider: 92%|#########1| 275/300 [1:28:00<07:26, 17.86s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 92%|#########1| 275/300 [1:28:00<07:26, 17.86s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 92%|#########2| 276/300 [1:28:00<05:51, 14.63s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 92%|#########2| 276/300 [1:28:00<05:51, 14.63s/it, lr: 1.0e-03 loss: 1.721e-02] The_Cosmic_Drift_Slider: 92%|#########2| 276/300 [1:28:13<05:51, 14.63s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 92%|#########2| 276/300 [1:28:13<05:51, 14.63s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 92%|#########2| 277/300 [1:28:13<05:26, 14.20s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 92%|#########2| 277/300 [1:28:13<05:26, 14.20s/it, lr: 1.0e-03 loss: 1.123e-02] The_Cosmic_Drift_Slider: 92%|#########2| 277/300 [1:28:34<05:26, 14.20s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 92%|#########2| 277/300 [1:28:34<05:26, 14.20s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 93%|#########2| 278/300 [1:28:34<05:49, 15.89s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 93%|#########2| 278/300 [1:28:34<05:49, 15.89s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 93%|#########2| 278/300 [1:28:55<05:49, 15.89s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 93%|#########2| 278/300 [1:28:55<05:49, 15.89s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 93%|#########3| 279/300 [1:28:55<06:01, 17.22s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 93%|#########3| 279/300 [1:28:55<06:01, 17.22s/it, lr: 1.0e-03 loss: 1.758e-02] The_Cosmic_Drift_Slider: 93%|#########3| 279/300 [1:29:17<06:01, 17.22s/it, lr: 1.0e-03 loss: 2.380e-02] The_Cosmic_Drift_Slider: 93%|#########3| 279/300 [1:29:17<06:01, 17.22s/it, lr: 1.0e-03 loss: 2.380e-02] The_Cosmic_Drift_Slider: 93%|#########3| 280/300 [1:29:38<05:44, 17.22s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 93%|#########3| 280/300 [1:29:38<05:44, 17.22s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 94%|#########3| 281/300 [1:29:38<04:31, 14.28s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 94%|#########3| 281/300 [1:29:38<04:31, 14.28s/it, lr: 1.0e-03 loss: 1.685e-02] The_Cosmic_Drift_Slider: 94%|#########3| 281/300 [1:29:50<04:31, 14.28s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 94%|#########3| 281/300 [1:29:50<04:31, 14.28s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 94%|#########3| 282/300 [1:29:50<04:10, 13.91s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 94%|#########3| 282/300 [1:29:50<04:10, 13.91s/it, lr: 1.0e-03 loss: 1.477e-02] The_Cosmic_Drift_Slider: 94%|#########3| 282/300 [1:30:11<04:10, 13.91s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 94%|#########3| 282/300 [1:30:11<04:10, 13.91s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 94%|#########4| 283/300 [1:30:11<04:26, 15.65s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 94%|#########4| 283/300 [1:30:11<04:26, 15.65s/it, lr: 1.0e-03 loss: 1.953e-02] The_Cosmic_Drift_Slider: 94%|#########4| 283/300 [1:30:33<04:26, 15.65s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 94%|#########4| 283/300 [1:30:33<04:26, 15.65s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 95%|#########4| 284/300 [1:30:33<04:33, 17.06s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 95%|#########4| 284/300 [1:30:33<04:33, 17.06s/it, lr: 1.0e-03 loss: 1.807e-02] The_Cosmic_Drift_Slider: 95%|#########4| 284/300 [1:30:46<04:33, 17.06s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 95%|#########4| 284/300 [1:30:46<04:33, 17.06s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 95%|#########5| 285/300 [1:30:46<03:58, 15.88s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 95%|#########5| 285/300 [1:30:46<03:58, 15.88s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 95%|#########5| 285/300 [1:31:06<03:58, 15.88s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 95%|#########5| 285/300 [1:31:06<03:58, 15.88s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 95%|#########5| 286/300 [1:31:06<04:01, 17.28s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 95%|#########5| 286/300 [1:31:06<04:01, 17.28s/it, lr: 1.0e-03 loss: 1.990e-02] The_Cosmic_Drift_Slider: 95%|#########5| 286/300 [1:31:27<04:01, 17.28s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 95%|#########5| 286/300 [1:31:27<04:01, 17.28s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 96%|#########5| 287/300 [1:31:27<03:57, 18.24s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 96%|#########5| 287/300 [1:31:27<03:57, 18.24s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 96%|#########5| 287/300 [1:31:49<03:57, 18.24s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 96%|#########5| 287/300 [1:31:49<03:57, 18.24s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 96%|#########6| 288/300 [1:31:49<03:48, 19.05s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 96%|#########6| 288/300 [1:31:49<03:48, 19.05s/it, lr: 1.0e-03 loss: 1.550e-02] The_Cosmic_Drift_Slider: 96%|#########6| 288/300 [1:32:02<03:48, 19.05s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 96%|#########6| 288/300 [1:32:02<03:48, 19.05s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 96%|#########6| 289/300 [1:32:02<03:09, 17.19s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 96%|#########6| 289/300 [1:32:02<03:09, 17.19s/it, lr: 1.0e-03 loss: 1.282e-02] The_Cosmic_Drift_Slider: 96%|#########6| 289/300 [1:32:22<03:09, 17.19s/it, lr: 1.0e-03 loss: 2.112e-02] The_Cosmic_Drift_Slider: 96%|#########6| 289/300 [1:32:22<03:09, 17.19s/it, lr: 1.0e-03 loss: 2.112e-02] The_Cosmic_Drift_Slider: 97%|#########6| 290/300 [1:32:43<02:51, 17.19s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 97%|#########6| 290/300 [1:32:43<02:51, 17.19s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 97%|#########7| 291/300 [1:32:43<02:07, 14.12s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 97%|#########7| 291/300 [1:32:43<02:07, 14.12s/it, lr: 1.0e-03 loss: 1.453e-02] The_Cosmic_Drift_Slider: 97%|#########7| 291/300 [1:32:57<02:07, 14.12s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 97%|#########7| 291/300 [1:32:57<02:07, 14.12s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 97%|#########7| 292/300 [1:32:57<01:50, 13.82s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 97%|#########7| 292/300 [1:32:57<01:50, 13.82s/it, lr: 1.0e-03 loss: 1.709e-02] The_Cosmic_Drift_Slider: 97%|#########7| 292/300 [1:33:18<01:50, 13.82s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 97%|#########7| 292/300 [1:33:18<01:50, 13.82s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 98%|#########7| 293/300 [1:33:18<01:49, 15.65s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 98%|#########7| 293/300 [1:33:18<01:49, 15.65s/it, lr: 1.0e-03 loss: 1.562e-02] The_Cosmic_Drift_Slider: 98%|#########7| 293/300 [1:33:39<01:49, 15.65s/it, lr: 1.0e-03 loss: 1.770e-02] The_Cosmic_Drift_Slider: 98%|#########7| 293/300 [1:33:39<01:49, 15.65s/it, lr: 1.0e-03 loss: 1.770e-02] The_Cosmic_Drift_Slider: 98%|#########8| 294/300 [1:33:39<01:42, 17.04s/it, lr: 1.0e-03 loss: 1.770e-02] The_Cosmic_Drift_Slider: 98%|#########8| 294/300 [1:33:39<01:42, 17.04s/it, lr: 1.0e-03 loss: 1.770e-02] The_Cosmic_Drift_Slider: 98%|#########8| 294/300 [1:34:00<01:42, 17.04s/it, lr: 1.0e-03 loss: 1.746e-02] The_Cosmic_Drift_Slider: 98%|#########8| 294/300 [1:34:00<01:42, 17.04s/it, lr: 1.0e-03 loss: 1.746e-02] The_Cosmic_Drift_Slider: 98%|#########8| 295/300 [1:34:00<01:30, 18.10s/it, lr: 1.0e-03 loss: 1.746e-02] The_Cosmic_Drift_Slider: 98%|#########8| 295/300 [1:34:00<01:30, 18.10s/it, lr: 1.0e-03 loss: 1.746e-02] The_Cosmic_Drift_Slider: 98%|#########8| 295/300 [1:34:21<01:30, 18.10s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 98%|#########8| 295/300 [1:34:21<01:30, 18.10s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 99%|#########8| 296/300 [1:34:21<01:15, 18.92s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 99%|#########8| 296/300 [1:34:21<01:15, 18.92s/it, lr: 1.0e-03 loss: 2.344e-02] The_Cosmic_Drift_Slider: 99%|#########8| 296/300 [1:34:42<01:15, 18.92s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 99%|#########8| 296/300 [1:34:42<01:15, 18.92s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 99%|#########9| 297/300 [1:34:42<00:58, 19.47s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 99%|#########9| 297/300 [1:34:42<00:58, 19.47s/it, lr: 1.0e-03 loss: 1.624e-02] The_Cosmic_Drift_Slider: 99%|#########9| 297/300 [1:34:55<00:58, 19.47s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 99%|#########9| 297/300 [1:34:55<00:58, 19.47s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 99%|#########9| 298/300 [1:34:55<00:34, 17.50s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 99%|#########9| 298/300 [1:34:55<00:34, 17.50s/it, lr: 1.0e-03 loss: 1.093e-02] The_Cosmic_Drift_Slider: 99%|#########9| 298/300 [1:35:16<00:34, 17.50s/it, lr: 1.0e-03 loss: 3.271e-02] The_Cosmic_Drift_Slider: 99%|#########9| 298/300 [1:35:16<00:34, 17.50s/it, lr: 1.0e-03 loss: 3.271e-02] The_Cosmic_Drift_Slider: 100%|#########9| 299/300 [1:35:16<00:18, 18.48s/it, lr: 1.0e-03 loss: 3.271e-02] The_Cosmic_Drift_Slider: 100%|#########9| 299/300 [1:35:16<00:18, 18.48s/it, lr: 1.0e-03 loss: 3.271e-02] The_Cosmic_Drift_Slider: 100%|#########9| 299/300 [1:35:16<00:19, 19.12s/it, lr: 1.0e-03 loss: 3.271e-02] The_Cosmic_Drift_Slider: 100%|#########9| 299/300 [1:35:16<00:19, 19.12s/it, lr: 1.0e-03 loss: 3.271e-02] | |
| Generating Images: 0%| | 0/7 [00:00<?, ?it/s] Generating Images: 0%| | 0/7 [00:00<?, ?it/s] Generating Images: 14%|#4 | 1/7 [01:37<09:46, 97.83s/it] Generating Images: 14%|#4 | 1/7 [01:37<09:46, 97.83s/it] Generating Images: 29%|##8 | 2/7 [03:15<08:08, 97.62s/it] Generating Images: 29%|##8 | 2/7 [03:15<08:08, 97.62s/it] Generating Images: 43%|####2 | 3/7 [04:52<06:29, 97.28s/it] Generating Images: 43%|####2 | 3/7 [04:52<06:29, 97.28s/it] Generating Images: 57%|#####7 | 4/7 [06:16<04:36, 92.25s/it] Generating Images: 57%|#####7 | 4/7 [06:16<04:36, 92.25s/it] Generating Images: 71%|#######1 | 5/7 [07:54<03:08, 94.21s/it] Generating Images: 71%|#######1 | 5/7 [07:54<03:08, 94.21s/it] Generating Images: 86%|########5 | 6/7 [09:31<01:35, 95.07s/it] Generating Images: 86%|########5 | 6/7 [09:31<01:35, 95.07s/it] Generating Images: 100%|##########| 7/7 [11:08<00:00, 95.83s/it] Generating Images: 100%|##########| 7/7 [11:08<00:00, 95.83s/it] | |
| Saved checkpoint to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/The_Cosmic_Drift_Slider.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/The_Cosmic_Drift_Slider/optimizer.pt | |