init

configs/models/qwen2_5_1_5b_radio_sd3_dynamic_puffin.py

@@ -7,7 +7,7 @@ from diffusers import AutoencoderKL, FlowMatchEulerDiscreteScheduler
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 llm_name_or_path = 'Qwen/Qwen2.5-1.5B-Instruct'
-sd3_model_name_or_path = "stabilityai/stable-diffusion-3-medium-diffusers"
+sd3_model_name_or_path = "configs/sd3"
 
 prompt_template = dict(
     SYSTEM=('<|im_start|>system\n{system}<|im_end|>\n'),
@@ -40,26 +40,26 @@ model = dict(type=Qwen2p5RadioStableDiffusion3HFDynamic,
                  num_attention_heads=16, 
                  ),
              transformer=dict(
-                 type=SD3Transformer2DModel.from_pretrained,
+                 type=SD3Transformer2DModel.from_config,
                  pretrained_model_name_or_path=sd3_model_name_or_path,
                  subfolder="transformer",
                  torch_dtype=torch.bfloat16,
                  #local_files_only=True,
                  ),
              test_scheduler=dict(
-                 type=FlowMatchEulerDiscreteScheduler.from_pretrained,
+                 type=FlowMatchEulerDiscreteScheduler.from_config,
                  pretrained_model_name_or_path=sd3_model_name_or_path,
                  subfolder="scheduler",
                  #local_files_only=True,
                  ),
              train_scheduler=dict(
-                 type=FlowMatchEulerDiscreteScheduler.from_pretrained,
+                 type=FlowMatchEulerDiscreteScheduler.from_config,
                  pretrained_model_name_or_path=sd3_model_name_or_path,
                  subfolder="scheduler",
                  #local_files_only=True,
                  ),
              vae=dict(
-                 type=AutoencoderKL.from_pretrained,
+                 type=AutoencoderKL.from_config,
                  pretrained_model_name_or_path=sd3_model_name_or_path,
                  subfolder="vae",
                  torch_dtype=torch.bfloat16,

configs/sd3/scheduler/scheduler_config.json

@@ -0,0 +1,6 @@
+{
+  "_class_name": "FlowMatchEulerDiscreteScheduler",
+  "_diffusers_version": "0.29.0.dev0",
+  "num_train_timesteps": 1000,
+  "shift": 3.0
+}

configs/sd3/transformer/config.json

@@ -0,0 +1,15 @@
+{
+  "_class_name": "SD3Transformer2DModel",
+  "_diffusers_version": "0.29.0.dev0",
+  "attention_head_dim": 64,
+  "caption_projection_dim": 1536,
+  "in_channels": 16,
+  "joint_attention_dim": 4096,
+  "num_attention_heads": 24,
+  "num_layers": 24,
+  "out_channels": 16,
+  "patch_size": 2,
+  "pooled_projection_dim": 2048,
+  "pos_embed_max_size": 192,
+  "sample_size": 128
+}

configs/sd3/vae/config.json

@@ -0,0 +1,36 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.29.0.dev0",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 16,
+  "latents_mean": null,
+  "latents_std": null,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 1024,
+  "scaling_factor": 1.5305,
+  "shift_factor": 0.0609,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ],
+  "use_post_quant_conv": false,
+  "use_quant_conv": false
+}