Upload 19 files

docs/bucket.txt

@@ -0,0 +1,92 @@
+bucket 0: resolution (256, 3328), count: 177     
+bucket 1: resolution (256, 3392), count: 20      
+bucket 2: resolution (256, 3456), count: 19      
+bucket 3: resolution (256, 3520), count: 9       
+bucket 4: resolution (256, 3584), count: 25      
+bucket 5: resolution (256, 3648), count: 22      
+bucket 6: resolution (256, 3712), count: 11      
+bucket 7: resolution (256, 3776), count: 16      
+bucket 8: resolution (256, 3840), count: 12      
+bucket 9: resolution (256, 3904), count: 12      
+bucket 10: resolution (256, 3968), count: 9      
+bucket 11: resolution (256, 4032), count: 12     
+bucket 12: resolution (256, 4096), count: 393    
+bucket 13: resolution (320, 2752), count: 309    
+bucket 14: resolution (320, 2816), count: 37     
+bucket 15: resolution (320, 2880), count: 45     
+bucket 16: resolution (320, 2944), count: 34     
+bucket 17: resolution (320, 3008), count: 25     
+bucket 18: resolution (320, 3072), count: 36     
+bucket 19: resolution (320, 3136), count: 46     
+bucket 20: resolution (320, 3200), count: 74     
+bucket 21: resolution (320, 3264), count: 185    
+bucket 22: resolution (384, 2368), count: 483    
+bucket 23: resolution (384, 2432), count: 87     
+bucket 24: resolution (384, 2496), count: 64     
+bucket 25: resolution (384, 2560), count: 81     
+bucket 26: resolution (384, 2624), count: 66     
+bucket 27: resolution (384, 2688), count: 282    
+bucket 28: resolution (448, 2112), count: 1185   
+bucket 29: resolution (448, 2176), count: 121    
+bucket 30: resolution (448, 2240), count: 191    
+bucket 31: resolution (448, 2304), count: 449    
+bucket 32: resolution (512, 1856), count: 1301   
+bucket 33: resolution (512, 1920), count: 323    
+bucket 34: resolution (512, 1984), count: 290    
+bucket 35: resolution (512, 2048), count: 872    
+bucket 36: resolution (576, 1664), count: 8969   
+bucket 37: resolution (576, 1728), count: 5854   
+bucket 38: resolution (576, 1792), count: 7437   
+bucket 39: resolution (640, 1536), count: 13187  
+bucket 40: resolution (640, 1600), count: 9038   
+bucket 41: resolution (704, 1408), count: 53679  
+bucket 42: resolution (704, 1472), count: 26199  
+bucket 43: resolution (768, 1280), count: 200880 
+bucket 44: resolution (768, 1344), count: 127072 
+bucket 45: resolution (832, 1216), count: 1351708
+bucket 46: resolution (896, 1152), count: 543197 
+bucket 47: resolution (960, 1088), count: 223327 
+bucket 48: resolution (1024, 1024), count: 306486
+bucket 49: resolution (1088, 960), count: 114596 
+bucket 50: resolution (1152, 896), count: 205301 
+bucket 51: resolution (1216, 832), count: 252788 
+bucket 52: resolution (1280, 768), count: 60456  
+bucket 53: resolution (1344, 768), count: 106301 
+bucket 54: resolution (1408, 704), count: 19124  
+bucket 55: resolution (1472, 704), count: 9013   
+bucket 56: resolution (1536, 640), count: 5820   
+bucket 57: resolution (1600, 640), count: 2820   
+bucket 58: resolution (1664, 576), count: 1632   
+bucket 59: resolution (1728, 576), count: 969    
+bucket 60: resolution (1792, 576), count: 709    
+bucket 61: resolution (1856, 512), count: 483    
+bucket 62: resolution (1920, 512), count: 101    
+bucket 63: resolution (1984, 512), count: 78     
+bucket 64: resolution (2048, 512), count: 288    
+bucket 65: resolution (2112, 448), count: 129    
+bucket 66: resolution (2176, 448), count: 44     
+bucket 67: resolution (2240, 448), count: 46     
+bucket 68: resolution (2304, 448), count: 95     
+bucket 69: resolution (2368, 384), count: 48     
+bucket 70: resolution (2432, 384), count: 11     
+bucket 71: resolution (2496, 384), count: 11     
+bucket 72: resolution (2560, 384), count: 13     
+bucket 73: resolution (2624, 384), count: 9      
+bucket 74: resolution (2688, 384), count: 32     
+bucket 75: resolution (2752, 320), count: 31     
+bucket 76: resolution (2816, 320), count: 1      
+bucket 77: resolution (2880, 320), count: 2      
+bucket 78: resolution (2944, 320), count: 4      
+bucket 79: resolution (3008, 320), count: 4      
+bucket 80: resolution (3072, 320), count: 4      
+bucket 81: resolution (3136, 320), count: 2      
+bucket 82: resolution (3200, 320), count: 8      
+bucket 83: resolution (3264, 320), count: 17     
+bucket 84: resolution (3328, 256), count: 7      
+bucket 85: resolution (3392, 256), count: 3      
+bucket 86: resolution (3456, 256), count: 1      
+bucket 87: resolution (3520, 256), count: 1      
+bucket 88: resolution (3712, 256), count: 2      
+bucket 89: resolution (3776, 256), count: 2      
+bucket 90: resolution (3840, 256), count: 2      
+bucket 91: resolution (4096, 256), count: 34     

docs/lyco_config.toml

@@ -0,0 +1,48 @@
+enable_conv = true
+
+# An example for use different algo/settings in "full" preset
+unet_target_module = [
+  "Transformer2DModel", 
+  "ResnetBlock2D", 
+  "Downsample2D", 
+  "Upsample2D",
+]
+unet_target_name = [
+  ".*time_embed\\..+",
+  ".*label_emb\\..+",
+  ".*input_blocks\\.0.+",
+  "^out\\..+"
+]
+text_encoder_target_module = [
+  "CLIPAttention",
+  "CLIPMLP",
+]
+text_encoder_target_name = [
+  # "token_embedding", # not supported, Embedding module in CLIP
+]
+
+
+[module_algo_map]
+  [module_algo_map.CrossAttention]  
+    algo = "lokr"
+    dim = 10000
+    alpha = 10000
+    factor = 4
+  
+  [module_algo_map.FeedForward]     # MLP Layer in UNet
+    algo = "lokr"
+    dim = 10000
+    alpha = 10000
+    factor = 2
+  
+  [module_algo_map.CLIPAttention]   # Attention Layer in TE
+    algo = "lokr"
+    dim = 10000
+    alpha = 10000
+    factor = 8
+  
+  [module_algo_map.CLIPMLP]   # MLP Layer in TE
+    algo = "lokr"
+    dim = 10000
+    alpha = 10000
+    factor = 8

docs/train_script.sh

@@ -0,0 +1,56 @@
+python3 -m accelerate.commands.launch \
+    --multi_gpu \
+    --num_processes=2 \
+    --num_cpu_threads_per_process=16 sdxl_train_network.py \
+    --enable_bucket \
+    --optimizer_type Lion8bit \
+    --save_state \
+    --network_args \
+        conv_dim=10000 \
+        conv_alpha=10000 \
+        algo=lokr \
+        dropout=0 \
+        factor=8 \
+        train_norm=True \
+        preset=./sdxl-base.toml \
+    --noise_offset 0.05 \
+    --min_snr_gamma 5 \
+    --pretrained_model_name_or_path=sd-models/kohaku-xl-delta-pre2.safetensors \
+    --train_data_dir=datasets/danbooru/kxl-delta \
+    --output_dir=output \
+    --logging_dir=./logs \
+    --log_prefix=kohaku-xl-delta-lyco \
+    --resolution=1024,1024 \
+    --network_module=lycoris.kohya \
+    --max_train_epochs=1 \
+    --save_every_n_steps=2000 \
+    --unet_lr=4e-5 \
+    --text_encoder_lr=1e-5 \
+    --lr_scheduler=constant_with_warmup \
+    --lr_warmup_steps=100 \
+    --lr_scheduler_num_cycles=1 \
+    --network_dim=10000 \
+    --network_alpha=10000 \
+    --output_name=kohaku-xl-delta-lyco \
+    --train_batch_size=4 \
+    --gradient_accumulation_steps=16 \
+    --save_every_n_epochs=1 \
+    --mixed_precision=fp16 \
+    --save_precision=fp16 \
+    --full_fp16 \
+    --seed=1337 \
+    --prior_loss_weight=1 \
+    --max_token_length=225 \
+    --caption_extension=.txt \
+    --save_model_as=safetensors \
+    --min_bucket_reso=256 \
+    --max_bucket_reso=4096 \
+    --keep_tokens=0 \
+    --xformers \
+    --gradient_checkpointing \
+    --shuffle_caption \
+    --caption_tag_dropout_rate=0.1 \
+    --keep_tokens_separator="|||" \
+    --optimizer_args weight_decay=0.1 betas=0.9,0.95 \
+    --ddp_gradient_as_bucket_view \
+    --ddp_timeout 10080

lyco-ckpts/kohaku-xl-delta-lyco-step00002000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9c149cc8434bde3ecabe57bd3e2a32776255051fabc262339c2c6d37fe2ca16
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00004000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb5a4d7aaf6d2ec00899ba1d69714c6c6ef07c8bef2e72b4ff8b7f3c95f637c0
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00006000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:316b8393ec70dc50e29167e54716e639b3e66242f2cdedeb8789018aec5cafed
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00008000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c456935595be57fe5bde82eef6af30df75c7033a3e83411db8df8c1424f437f3
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00010000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aada94008e97a42135d775ec3c331ef5724a1faa1c220700b1505313b89a43e1
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00012000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e36d0b85403024629da334d8c5a723f9dac10a2b3a94b297c5d9efb028cd578
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00014000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d82802ac04b66589db8c746ad466917cfbe2227eca0058ca43ef842476470cb
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00016000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df76e94bf87bccfdcb6445ad6404101b7c28c6ae656cbd9110b6b43ceac5abfe
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00018000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c89e706f8e6b261866c28bc205883571acecb34b661573f5f312480729543b7
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00020000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:467450b843bc0fdf49760e80c444afa5e6755896c6618d172b764e95a4a419f9
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00022000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ecf375b79692c39f98f4c0737a98aa89e938684d85e86634e3356372d1e39bb
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00024000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa327210f54614aeac65c3a4f57e882b479673c8472f880aa0394df83f9c4a71
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00026000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eb67e8d4414d1556906778c96b08c91b679e82c399ab6fd16b9d77063998147
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco-step00028000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51e828deacf45435dd33bea4f1e0140343042fdd819eed4aa70197f58b4f7d0e
+size 782617220

lyco-ckpts/kohaku-xl-delta-lyco.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1208d778405b50e768479eb577a1a0659d1c9630e5ac72a210ea02ba11b1adf4
+size 782617220