Update examples

app.py

@@ -88,7 +88,7 @@ def get_layout_image(response):
 def get_layout_image_gallery(response):
     return [get_layout_image(response)]
 
-def get_ours_image(response, seed, num_inference_steps=20, dpm_scheduler=True, use_autocast=False, overall_prompt_override="", fg_seed_start=20, fg_blending_ratio=0.1, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta=0.3, so_negative_prompt=DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt=DEFAULT_OVERALL_NEGATIVE_PROMPT, show_so_imgs=False, scale_boxes=False):
+def get_ours_image(response, overall_prompt_override="", seed=0, num_inference_steps=20, dpm_scheduler=True, use_autocast=False, fg_seed_start=20, fg_blending_ratio=0.1, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta=0.3, so_negative_prompt=DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt=DEFAULT_OVERALL_NEGATIVE_PROMPT, show_so_imgs=False, scale_boxes=False):
     if response == "":
         response = layout_placeholder
     gen_boxes, bg_prompt = parse_input(response)
@@ -121,8 +121,9 @@ def get_baseline_image(prompt, seed=0):
         prompt = prompt_placeholder
     
     scheduler_key = "dpm_scheduler"
+    num_inference_steps = 50
     
-    image_np = run_baseline(prompt, bg_seed=seed, scheduler_key=scheduler_key)
+    image_np = run_baseline(prompt, bg_seed=seed, scheduler_key=scheduler_key, num_inference_steps=num_inference_steps)
     return [image_np]
 
 def parse_input(text=None):
@@ -202,7 +203,7 @@ html = f"""<h1>LLM-grounded Diffusion: Enhancing Prompt Understanding of Text-to
             <p>2. You can perform multi-round specification by giving ChatGPT follow-up requests (e.g., make the object boxes bigger).</p>
             <p>3. You can also try prompts in Simplified Chinese. If you want to try prompts in another language, translate the first line of last example to your language.</p>
             <p>4. The diffusion model only runs 50 steps by default in this demo. You can make it run more/fewer steps to get higher quality images or faster generation (or tweak frozen steps/guidance steps for better guidance and coherence).</p>
-            <p>5. Duplicate this space and add GPU or clone the space and run locally to skip the queue and run our model faster. (Currently we are using a T4, and you can add a A10G to make it 5x faster) {duplicate_html}</p>
+            <p>5. Duplicate this space and add GPU or clone the space and run locally to skip the queue and run our model faster. (<b>Currently we are using a T4 GPU on this space, which is quite slow, and you can add a A10G to make it 5x faster</b>) {duplicate_html}</p>
             <br/>
             <p>Implementation note: In this demo, we replace the attention manipulation in our layout-guided Stable Diffusion described in our paper with GLIGEN due to much faster inference speed (<b>FlashAttention supported, no backprop needed</b> during inference). Compared to vanilla GLIGEN, we have better coherence. Other parts of text-to-image pipeline, including single object generation and SAM, remain the same. The settings and examples in the prompt are simplified in this demo.</p>
             <style>.btn {{flex-grow: unset !important;}} </style>
@@ -243,7 +244,7 @@ with gr.Blocks(
                     frozen_step_ratio = gr.Slider(0, 1, value=0.4, step=0.1, label="Foreground frozen steps ratio (higher: preserve object attributes; lower: higher coherence; set to 0: (almost) equivalent to vanilla GLIGEN except details)")
                     gligen_scheduled_sampling_beta = gr.Slider(0, 1, value=0.3, step=0.1, label="GLIGEN guidance steps ratio (the beta value)")
                     dpm_scheduler = gr.Checkbox(label="Use DPM scheduler (unchecked: DDIM scheduler, may have better coherence, recommend >=50 inference steps)", show_label=False, value=True)
-                    use_autocast = gr.Checkbox(label="Use FP16 Mixed Precision", show_label=False, value=True)
+                    use_autocast = gr.Checkbox(label="Use FP16 Mixed Precision (faster but with slightly lower quality)", show_label=False, value=True)
                     fg_seed_start = gr.Slider(0, 10000, value=20, step=1, label="Seed for foreground variation")
                     fg_blending_ratio = gr.Slider(0, 1, value=0.1, step=0.01, label="Variations added to foreground for single object generation (0: no variation, 1: max variation)")
                     so_negative_prompt = gr.Textbox(lines=1, label="Negative prompt for single object generation", value=DEFAULT_SO_NEGATIVE_PROMPT)
@@ -257,11 +258,11 @@ with gr.Blocks(
                     label="Generated image", show_label=False, elem_id="gallery", columns=[1], rows=[1], object_fit="contain", preview=True
                 )
         visualize_btn.click(fn=get_layout_image_gallery, inputs=response, outputs=gallery, api_name="visualize-layout")
-        generate_btn.click(fn=get_ours_image, inputs=[response, seed, num_inference_steps, dpm_scheduler, use_autocast, overall_prompt_override, fg_seed_start, fg_blending_ratio, frozen_step_ratio, gligen_scheduled_sampling_beta, so_negative_prompt, overall_negative_prompt, show_so_imgs, scale_boxes], outputs=gallery, api_name="layout-to-image")
+        generate_btn.click(fn=get_ours_image, inputs=[response, overall_prompt_override, seed, num_inference_steps, dpm_scheduler, use_autocast, fg_seed_start, fg_blending_ratio, frozen_step_ratio, gligen_scheduled_sampling_beta, so_negative_prompt, overall_negative_prompt, show_so_imgs, scale_boxes], outputs=gallery, api_name="layout-to-image")
 
         gr.Examples(
             examples=stage2_examples,
-            inputs=[response, seed],
+            inputs=[response, overall_prompt_override, seed],
             outputs=[gallery],
             fn=get_ours_image,
             cache_examples=True

examples.py

@@ -2,7 +2,7 @@ stage1_examples = [
     ["""A realistic photo of a gray cat and an orange dog on the grass."""],
     ["""In an indoor scene, a blue cube directly above a red cube with a vase on the left of them."""],
     ["""A realistic photo of a wooden table without bananas in an indoor scene"""],
-    ["""A man in red is standing next to another woman in blue in the mountains."""],
+    ["""A realistic photo of two cars on the road."""],
     ["""一个室内场景的水彩画，一个桌子上面放着一盘水果"""]
 ]
 
@@ -10,17 +10,17 @@ stage1_examples = [
 stage2_examples = [
     ["""Caption: A realistic photo of a gray cat and an orange dog on the grass.
 Objects: [('a gray cat', [67, 243, 120, 126]), ('an orange dog', [265, 193, 190, 210])]
-Background prompt: A realistic photo of a grassy area.""", 0],
+Background prompt: A realistic photo of a grassy area.""", "", 0],
     ["""Caption: 一个室内场景的水彩画，一个桌子上面放着一盘水果
 Objects: [('a table', [81, 242, 350, 210]), ('a plate of fruits', [151, 287, 210, 117])]
-Background prompt: A watercolor painting of an indoor scene""", 1],
+Background prompt: A watercolor painting of an indoor scene""", "", 1],
     ["""Caption: In an indoor scene, a blue cube directly above a red cube with a vase on the left of them.
 Objects: [('a blue cube', [232, 116, 76, 76]), ('a red cube', [232, 212, 76, 76]), ('a vase', [100, 198, 62, 144])]
-Background prompt: An indoor scene""", 2],
+Background prompt: An indoor scene""", "", 2],
     ["""Caption: A realistic photo of a wooden table without bananas in an indoor scene
 Objects: [('a wooden table', [75, 256, 365, 156])]
-Background prompt: A realistic photo of an indoor scene""", 3],
-    ["""Caption: A man in red is standing next to another woman in blue in the mountains.
-Objects: [('a man in red', [100, 160, 111, 320]), ('a woman in blue', [230, 170, 102, 310])]
-Background prompt: A scenic image of the mountains""", 4],
+Background prompt: A realistic photo of an indoor scene""", "", 3],
+    ["""Caption: A realistic photo of two cars on the road.
+Objects: [('a car', [20, 242, 235, 185]), ('a car', [275, 246, 215, 180])]
+Background prompt: A realistic photo of a road.""", "A realistic photo of two cars on the road.", 4],
 ]

gradio_cached_examples/15/log.csv

@@ -30,7 +30,7 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: A realistic photo of a gray cat and an orange dog on the grass.
-Objects: ",,,2023-06-19 12:19:18.120678
+Objects: ",,,2023-06-29 11:10:05.644376
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -62,7 +62,7 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: In an indoor scene, a blue cube directly above a red cube with a vase on the left of them.
-Objects: ",,,2023-06-19 12:19:18.121279
+Objects: ",,,2023-06-29 11:10:05.644933
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -94,7 +94,7 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: A realistic photo of a wooden table without bananas in an indoor scene
-Objects: ",,,2023-06-19 12:19:18.121771
+Objects: ",,,2023-06-29 11:10:05.645428
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -125,8 +125,8 @@ Caption: 一个客厅场景的油画，墙上挂着电视，电视下面是一
 Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a flower vase', [166, 222, 92, 108])]
 Background prompt: An oil painting of a living room scene
 
-Caption: A man in red is standing next to another woman in blue in the mountains.
-Objects: ",,,2023-06-19 12:19:18.122219
+Caption: A realistic photo of two cars on the road.
+Objects: ",,,2023-06-29 11:10:05.645947
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -158,4 +158,4 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: 一个室内场景的水彩画，一个桌子上面放着一盘水果
-Objects: ",,,2023-06-19 12:19:18.122722
+Objects: ",,,2023-06-29 11:10:05.646731

gradio_cached_examples/40/Generated image/2780fc35-353f-42b3-b738-86e456fe6f56/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/2780fc35-353f-42b3-b738-86e456fe6f56/2e40cc791e626d4b53246213fbb411de998a0c48/image.png": null}

gradio_cached_examples/40/Generated image/2b8c8fa6-273e-4993-b984-ca766b7fa093/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/40/Generated image/2b8c8fa6-273e-4993-b984-ca766b7fa093/6b8454f7801357c4c41076b4281ff36cc3c168d1/image.png": null}

gradio_cached_examples/40/Generated image/5cc41c19-e968-478e-b144-5b8ba3d07be1/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/40/Generated image/5cc41c19-e968-478e-b144-5b8ba3d07be1/65c5ef22af85300a0da31b44b1ad679a47eecc25/image.png": null}

gradio_cached_examples/40/Generated image/81f59ccb-1a36-42ae-9914-2552463399d2/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/81f59ccb-1a36-42ae-9914-2552463399d2/5da6114e92c338adc17812a0333df55a4234c9ed/image.png": null}

gradio_cached_examples/40/Generated image/ccb2d58b-c9c4-466f-a20e-360535ae287a/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/ccb2d58b-c9c4-466f-a20e-360535ae287a/d49483d8de74eb5fc4d6adfe89475c71d8e3036f/image.png": null}

gradio_cached_examples/40/Generated image/cd999256-9a24-4a5b-a897-5c5fa56c5a67/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/40/Generated image/cd999256-9a24-4a5b-a897-5c5fa56c5a67/101f36d55ac3f9ebcf7ff5647da0d7acb4994c97/image.png": null}

gradio_cached_examples/40/Generated image/e4db9a87-7616-4e70-b7f0-90d12815524a/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/e4db9a87-7616-4e70-b7f0-90d12815524a/b6ffdf54ecc5f212c05fea1d31937c56d9a46ed0/image.png": null}

gradio_cached_examples/40/Generated image/e5bbe48b-6f7d-45c9-b75f-38f1709e0026/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/40/Generated image/e5bbe48b-6f7d-45c9-b75f-38f1709e0026/210e77149171a376e7be7a733564930e356849db/image.png": null}

gradio_cached_examples/40/Generated image/eb50d204-9a80-4e66-82a4-b24be17a05c2/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/40/Generated image/eb50d204-9a80-4e66-82a4-b24be17a05c2/44f995a0ec0c935f5881edd8af7f10e857e1a968/image.png": null}

gradio_cached_examples/40/Generated image/f613c4a4-afa5-4849-8d9b-6e1422dab503/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/f613c4a4-afa5-4849-8d9b-6e1422dab503/a513ea281318c5da990ab8817d346efad4afb0ad/image.png": null}

gradio_cached_examples/40/log.csv

@@ -1,6 +1,6 @@
 Generated image,flag,username,timestamp
-./gradio_cached_examples/40/Generated image/5cc41c19-e968-478e-b144-5b8ba3d07be1,,,2023-06-20 07:17:18.077779
-./gradio_cached_examples/40/Generated image/2b8c8fa6-273e-4993-b984-ca766b7fa093,,,2023-06-20 07:17:23.197795
-./gradio_cached_examples/40/Generated image/cd999256-9a24-4a5b-a897-5c5fa56c5a67,,,2023-06-20 07:17:30.064619
-./gradio_cached_examples/40/Generated image/eb50d204-9a80-4e66-82a4-b24be17a05c2,,,2023-06-20 07:17:33.414443
-./gradio_cached_examples/40/Generated image/e5bbe48b-6f7d-45c9-b75f-38f1709e0026,,,2023-06-20 07:17:38.498364
+./gradio_cached_examples/40/Generated image/2780fc35-353f-42b3-b738-86e456fe6f56,,,2023-06-29 11:10:11.840068
+./gradio_cached_examples/40/Generated image/f613c4a4-afa5-4849-8d9b-6e1422dab503,,,2023-06-29 11:10:16.511970
+./gradio_cached_examples/40/Generated image/e4db9a87-7616-4e70-b7f0-90d12815524a,,,2023-06-29 11:10:22.574699
+./gradio_cached_examples/40/Generated image/81f59ccb-1a36-42ae-9914-2552463399d2,,,2023-06-29 11:10:25.942870
+./gradio_cached_examples/40/Generated image/ccb2d58b-c9c4-466f-a20e-360535ae287a,,,2023-06-29 11:10:30.618947

gradio_cached_examples/50/Generated image/12d9118b-abed-45a7-9114-6f50b13dde08/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/12d9118b-abed-45a7-9114-6f50b13dde08/574f77a943affe0685e630d003d40486d687548b/image.png": null}

gradio_cached_examples/50/Generated image/2b20f451-c8d6-4f26-b580-85a54989b5e7/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/50/Generated image/2b20f451-c8d6-4f26-b580-85a54989b5e7/e73c626c4dd21a7ad80767910261adf9442952fc/image.png": null}

gradio_cached_examples/50/Generated image/3506257d-6ff8-48f7-9ca2-88a1fa88a7a4/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/3506257d-6ff8-48f7-9ca2-88a1fa88a7a4/096597e5396f718c54fd1467ecbf8d64ed1c4b58/image.png": null}

gradio_cached_examples/50/Generated image/3ef81752-e136-466d-b5d0-fa2ddec37971/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/50/Generated image/3ef81752-e136-466d-b5d0-fa2ddec37971/6863bcdf1ddd78ec6b38cb7f7e54dfd02042cfc8/image.png": null}

gradio_cached_examples/50/Generated image/460ea7e5-92af-4b13-9c5d-1386164789d7/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/50/Generated image/460ea7e5-92af-4b13-9c5d-1386164789d7/c881ad29f622a8f1d9a88854d93ec05142de78c0/image.png": null}

gradio_cached_examples/50/Generated image/5f67e14e-c421-4885-837b-c089dd8f885b/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/5f67e14e-c421-4885-837b-c089dd8f885b/7206c6d00d3a9cfea4bae3cb8504c2170eb97799/image.png": null}

gradio_cached_examples/50/Generated image/7ce72b1f-5123-44fe-9eb8-723d43cdfbb3/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/7ce72b1f-5123-44fe-9eb8-723d43cdfbb3/af91a254a7889ad4a7f59df784a89add68b8bb04/image.png": null}

gradio_cached_examples/50/Generated image/8c451715-7470-46fc-9634-e16f88e98e49/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/50/Generated image/8c451715-7470-46fc-9634-e16f88e98e49/5107323699a702f3dbb8cac7480802bd1ce00aa8/image.png": null}

gradio_cached_examples/50/Generated image/bbaa5527-7cf2-47e4-aa03-6d68ecbaa65b/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/50/Generated image/bbaa5527-7cf2-47e4-aa03-6d68ecbaa65b/62563c47f44f246d810891c182da583bab12f8a0/image.png": null}

gradio_cached_examples/50/Generated image/c9a1be01-ac11-43c2-bb8f-b7eb5e1ac1c3/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/c9a1be01-ac11-43c2-bb8f-b7eb5e1ac1c3/9bcc5a4a5fa197042b7eb3ae9925bb6a1f3de0db/image.png": null}

gradio_cached_examples/50/log.csv

@@ -1,6 +1,6 @@
 Generated image,flag,username,timestamp
-./gradio_cached_examples/50/Generated image/3ef81752-e136-466d-b5d0-fa2ddec37971,,,2023-06-20 07:17:39.853483
-./gradio_cached_examples/50/Generated image/460ea7e5-92af-4b13-9c5d-1386164789d7,,,2023-06-20 07:17:41.227531
-./gradio_cached_examples/50/Generated image/bbaa5527-7cf2-47e4-aa03-6d68ecbaa65b,,,2023-06-20 07:17:42.621290
-./gradio_cached_examples/50/Generated image/2b20f451-c8d6-4f26-b580-85a54989b5e7,,,2023-06-20 07:17:43.974086
-./gradio_cached_examples/50/Generated image/8c451715-7470-46fc-9634-e16f88e98e49,,,2023-06-20 07:17:45.349694
+./gradio_cached_examples/50/Generated image/3506257d-6ff8-48f7-9ca2-88a1fa88a7a4,,,2023-06-29 11:10:33.709651
+./gradio_cached_examples/50/Generated image/5f67e14e-c421-4885-837b-c089dd8f885b,,,2023-06-29 11:10:36.806117
+./gradio_cached_examples/50/Generated image/7ce72b1f-5123-44fe-9eb8-723d43cdfbb3,,,2023-06-29 11:10:39.927271
+./gradio_cached_examples/50/Generated image/c9a1be01-ac11-43c2-bb8f-b7eb5e1ac1c3,,,2023-06-29 11:10:42.986167
+./gradio_cached_examples/50/Generated image/12d9118b-abed-45a7-9114-6f50b13dde08,,,2023-06-29 11:10:46.101567