demo

app.py

@@ -28,7 +28,7 @@ def main():
     model = RegionDiffusion(device)
 
     def generate(
-        text_input: str,
+        json.loads(text_input): str,
         negative_text: str,
         height: int,
         width: int,
@@ -44,7 +44,7 @@ def main():
         # parse json to span attributes
         base_text_prompt, style_text_prompts, footnote_text_prompts, footnote_target_tokens,\
             color_text_prompts, color_names, color_rgbs, size_text_prompts_and_sizes, use_grad_guidance = parse_json(
-                text_input)
+                json.loads(text_input))
 
         # create control input for region diffusion
         region_text_prompts, region_target_token_ids, base_tokens = get_region_diffusion_input(

models/region_diffusion.py

@@ -22,17 +22,12 @@ class RegionDiffusion(nn.Module):
         print(f'[INFO] loading stable diffusion...')
         model_id = 'runwayml/stable-diffusion-v1-5'
 
-        # 1. Load the autoencoder model which will be used to decode the latents into image space.
         self.vae = AutoencoderKL.from_pretrained(
             model_id, subfolder="vae").to(self.device)
-
-        # 2. Load the tokenizer and text encoder to tokenize and encode the text.
         self.tokenizer = CLIPTokenizer.from_pretrained(
             model_id, subfolder='tokenizer')
         self.text_encoder = CLIPTextModel.from_pretrained(
             model_id, subfolder='text_encoder').to(self.device)
-
-        # 3. The UNet model for generating the latents.
         self.unet = UNet2DConditionModel.from_pretrained(
             model_id, subfolder="unet").to(self.device)