Allow overriding the overall prompt

app.py

@@ -88,7 +88,7 @@ def get_layout_image(response):
 def get_layout_image_gallery(response):
     return [get_layout_image(response)]
 
-def get_ours_image(response, seed, num_inference_steps=20, dpm_scheduler=True, fg_seed_start=20, fg_blending_ratio=0.1, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta=0.3, so_negative_prompt=DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt=DEFAULT_OVERALL_NEGATIVE_PROMPT, show_so_imgs=False, scale_boxes=False):
+def get_ours_image(response, seed, num_inference_steps=20, dpm_scheduler=True, overall_prompt_override="", fg_seed_start=20, fg_blending_ratio=0.1, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta=0.3, so_negative_prompt=DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt=DEFAULT_OVERALL_NEGATIVE_PROMPT, show_so_imgs=False, scale_boxes=False):
     if response == "":
         response = layout_placeholder
     gen_boxes, bg_prompt = parse_input(response)
@@ -106,7 +106,7 @@ def get_ours_image(response, seed, num_inference_steps=20, dpm_scheduler=True, f
         scheduler_key = "scheduler"
         
     image_np, so_img_list = run_ours(
-        spec, bg_seed=seed, fg_seed_start=fg_seed_start, 
+        spec, bg_seed=seed, overall_prompt_override=overall_prompt_override, fg_seed_start=fg_seed_start, 
         fg_blending_ratio=fg_blending_ratio,frozen_step_ratio=frozen_step_ratio, 
         gligen_scheduled_sampling_beta=gligen_scheduled_sampling_beta, num_inference_steps=num_inference_steps, scheduler_key=scheduler_key,
         so_negative_prompt=so_negative_prompt, overall_negative_prompt=overall_negative_prompt
@@ -245,13 +245,14 @@ with gr.Blocks(
                 visualize_btn = gr.Button("Visualize Layout")
                 generate_btn = gr.Button("Generate Image from Layout", variant='primary')
                 with gr.Accordion("Advanced options (play around for better generation)", open=False):
+                    overall_prompt_override = gr.Textbox(lines=2, label="Prompt for overall generation (you can put your input prompt for layout generation here, helpful if your scene cannot be represented by background prompt and boxes, such as with object interactions; if left empty: background prompt with [objects])", value="")
+                    frozen_step_ratio = gr.Slider(0, 1, value=0.4, step=0.1, label="Foreground frozen steps ratio (higher: preserve object attributes; lower: higher coherence; set to 0: (almost) equivalent to vanilla GLIGEN except details)")
+                    gligen_scheduled_sampling_beta = gr.Slider(0, 1, value=0.3, step=0.1, label="GLIGEN guidance steps ratio (the beta value)")
                     seed = gr.Slider(0, 10000, value=0, step=1, label="Seed")
                     num_inference_steps = gr.Slider(1, 50, value=20, step=1, label="Number of inference steps")
                     dpm_scheduler = gr.Checkbox(label="Use DPM scheduler (unchecked: DDIM scheduler, may have better coherence, recommend 50 inference steps)", show_label=False, value=True)
                     fg_seed_start = gr.Slider(0, 10000, value=20, step=1, label="Seed for foreground variation")
                     fg_blending_ratio = gr.Slider(0, 1, value=0.1, step=0.01, label="Variations added to foreground for single object generation (0: no variation, 1: max variation)")
-                    frozen_step_ratio = gr.Slider(0, 1, value=0.4, step=0.1, label="Foreground frozen steps ratio (higher: preserve object attributes; lower: higher coherence; set to 0: (almost) equivalent to vanilla GLIGEN except details)")
-                    gligen_scheduled_sampling_beta = gr.Slider(0, 1, value=0.3, step=0.1, label="GLIGEN guidance steps ratio (the beta value)")
                     so_negative_prompt = gr.Textbox(lines=1, label="Negative prompt for single object generation", value=DEFAULT_SO_NEGATIVE_PROMPT)
                     overall_negative_prompt = gr.Textbox(lines=1, label="Negative prompt for overall generation", value=DEFAULT_OVERALL_NEGATIVE_PROMPT)
                     show_so_imgs = gr.Checkbox(label="Show annotated single object generations", show_label=False, value=False)
@@ -261,7 +262,7 @@ with gr.Blocks(
                     label="Generated image", show_label=False, elem_id="gallery"
                 ).style(columns=[1], rows=[1], object_fit="contain", preview=True)
         visualize_btn.click(fn=get_layout_image_gallery, inputs=response, outputs=gallery, api_name="visualize-layout")
-        generate_btn.click(fn=get_ours_image, inputs=[response, seed, num_inference_steps, dpm_scheduler, fg_seed_start, fg_blending_ratio, frozen_step_ratio, gligen_scheduled_sampling_beta, so_negative_prompt, overall_negative_prompt, show_so_imgs, scale_boxes], outputs=gallery, api_name="layout-to-image")
+        generate_btn.click(fn=get_ours_image, inputs=[response, seed, num_inference_steps, dpm_scheduler, overall_prompt_override, fg_seed_start, fg_blending_ratio, frozen_step_ratio, gligen_scheduled_sampling_beta, so_negative_prompt, overall_negative_prompt, show_so_imgs, scale_boxes], outputs=gallery, api_name="layout-to-image")
 
         gr.Examples(
             examples=stage2_examples,

generation.py

@@ -75,7 +75,7 @@ def get_masked_latents_all_list(so_prompt_phrase_word_box_list, input_latents_li
 # Note: need to keep the supervision, especially the box corrdinates, corresponds to each other in single object and overall.
 
 def run(
-    spec, bg_seed = 1, fg_seed_start = 20, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta = 0.3, num_inference_steps = 20,
+    spec, bg_seed = 1, overall_prompt_override="", fg_seed_start = 20, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta = 0.3, num_inference_steps = 20,
     so_center_box = False, fg_blending_ratio = 0.1, scheduler_key='dpm_scheduler', so_negative_prompt = DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt = DEFAULT_OVERALL_NEGATIVE_PROMPT, so_horizontal_center_only = True,
     align_with_overall_bboxes = False, horizontal_shift_only = True
 ):
@@ -95,6 +95,9 @@ def run(
     if True:
         so_prompt_phrase_word_box_list, overall_prompt, overall_phrases_words_bboxes = parse.convert_spec(spec, height, width, verbose=verbose)
 
+    if overall_prompt_override and overall_prompt_override.strip():
+        overall_prompt = overall_prompt_override.strip()
+
     overall_phrases, overall_words, overall_bboxes = [item[0] for item in overall_phrases_words_bboxes], [item[1] for item in overall_phrases_words_bboxes], [item[2] for item in overall_phrases_words_bboxes]
 
     # The so box is centered but the overall boxes are not (since we need to place to the right place).

gradio_cached_examples/14/log.csv

@@ -30,7 +30,7 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: A realistic photo of a gray cat and an orange dog on the grass.
-Objects: ",,,2023-06-13 20:35:31.473564
+Objects: ",,,2023-06-15 09:35:52.218273
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -62,7 +62,7 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: In an indoor scene, a blue cube directly above a red cube with a vase on the left of them.
-Objects: ",,,2023-06-13 20:35:31.474147
+Objects: ",,,2023-06-15 09:35:52.218791
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -94,7 +94,7 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: A realistic photo of a wooden table without bananas in an indoor scene
-Objects: ",,,2023-06-13 20:35:31.474590
+Objects: ",,,2023-06-15 09:35:52.219289
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -126,7 +126,7 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: A man in red is standing next to another woman in blue in the mountains.
-Objects: ",,,2023-06-13 20:35:31.475074
+Objects: ",,,2023-06-15 09:35:52.219742
 "You are an intelligent bounding box generator. I will provide you with a caption for a photo, image, or painting. Your task is to generate the bounding boxes for the objects mentioned in the caption, along with a background prompt describing the scene. The images are of size 512x512, and the bounding boxes should not overlap or go beyond the image boundaries. Each bounding box should be in the format of (object name, [top-left x coordinate, top-left y coordinate, box width, box height]) and include exactly one object. Make the boxes larger if possible. Do not put objects that are already provided in the bounding boxes into the background prompt. If needed, you can make reasonable guesses. Generate the object descriptions and background prompts in English even if the caption might not be in English. Do not include non-existing or excluded objects in the background prompt. Please refer to the example below for the desired format.
 
 Caption: A realistic image of landscape scene depicting a green car parking on the left of a blue truck, with a red air balloon and a bird in the sky
@@ -158,4 +158,4 @@ Objects: [('a tv', [88, 85, 335, 203]), ('a cabinet', [57, 308, 404, 201]), ('a
 Background prompt: An oil painting of a living room scene
 
 Caption: 一个室内场景的水彩画，一个桌子上面放着一盘水果
-Objects: ",,,2023-06-13 20:35:31.475563
+Objects: ",,,2023-06-15 09:35:52.220406

gradio_cached_examples/37/Generated image/2a52e23e-0f01-49ed-ad14-2013a907d46a/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/37/Generated image/2a52e23e-0f01-49ed-ad14-2013a907d46a/5bfcd12c591545c7eb38d8675608948b57813890/image.png": null}

gradio_cached_examples/37/Generated image/365c0a30-01a7-456d-90e2-0db9861d400f/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/37/Generated image/365c0a30-01a7-456d-90e2-0db9861d400f/298e05cc7b29916761eae5d8c549ebd8a9abc14e/image.png": null}

gradio_cached_examples/37/Generated image/3c50eff8-950e-45df-b001-67eff6adf17e/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/37/Generated image/3c50eff8-950e-45df-b001-67eff6adf17e/bbbcccd9a1784845351c7470dc409a041c45869c/image.png": null}

gradio_cached_examples/37/Generated image/c1cb09f3-b74a-41d4-8e8b-3f06aa4b4a59/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/37/Generated image/c1cb09f3-b74a-41d4-8e8b-3f06aa4b4a59/8e16b1c0cfa4398722069b18606c624716ad2847/image.png": null}

gradio_cached_examples/37/Generated image/c91c620c-5aea-444d-a3b4-ce10eb7d8096/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/37/Generated image/c91c620c-5aea-444d-a3b4-ce10eb7d8096/56f0a33343fbb9e150cb0934b608d817148b978a/image.png": null}

gradio_cached_examples/37/log.csv

@@ -1,6 +0,0 @@
-Generated image,flag,username,timestamp
-./gradio_cached_examples/37/Generated image/365c0a30-01a7-456d-90e2-0db9861d400f,,,2023-06-13 20:35:38.002200
-./gradio_cached_examples/37/Generated image/c1cb09f3-b74a-41d4-8e8b-3f06aa4b4a59,,,2023-06-13 20:35:43.328015
-./gradio_cached_examples/37/Generated image/2a52e23e-0f01-49ed-ad14-2013a907d46a,,,2023-06-13 20:35:50.185410
-./gradio_cached_examples/37/Generated image/c91c620c-5aea-444d-a3b4-ce10eb7d8096,,,2023-06-13 20:35:53.536056
-./gradio_cached_examples/37/Generated image/3c50eff8-950e-45df-b001-67eff6adf17e,,,2023-06-13 20:35:58.617592

gradio_cached_examples/38/Generated image/148d3dde-4179-4292-b993-143fd15e9398/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/38/Generated image/148d3dde-4179-4292-b993-143fd15e9398/5925a9dc4b13242116df32682bf6a18fc82b8176/image.png": null}

gradio_cached_examples/38/Generated image/1547ec6a-5af3-49dd-9b68-d08c023ad8f8/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/38/Generated image/1547ec6a-5af3-49dd-9b68-d08c023ad8f8/e3839730ea5d0b91f661a68669ed0803325bacc4/image.png": null}

gradio_cached_examples/38/Generated image/60d193e4-3d27-4007-9120-6a96c6678db9/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/38/Generated image/60d193e4-3d27-4007-9120-6a96c6678db9/e90243b7c539e93f3906354f759e117b55c35a18/image.png": null}

gradio_cached_examples/38/Generated image/9dbee7af-40cb-4416-ba6d-4a051fa8edb3/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/38/Generated image/9dbee7af-40cb-4416-ba6d-4a051fa8edb3/2efc85ea287574205da430f9450f330835b7e514/image.png": null}

gradio_cached_examples/38/Generated image/dbb8883c-9107-42f0-8e5d-01119e1898d8/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/38/Generated image/dbb8883c-9107-42f0-8e5d-01119e1898d8/706e488f6c9f497e763e653f3b923f5fde09c790/image.png": null}

gradio_cached_examples/38/log.csv

@@ -0,0 +1,6 @@
+Generated image,flag,username,timestamp
+./gradio_cached_examples/38/Generated image/dbb8883c-9107-42f0-8e5d-01119e1898d8,,,2023-06-15 09:35:58.721429
+./gradio_cached_examples/38/Generated image/1547ec6a-5af3-49dd-9b68-d08c023ad8f8,,,2023-06-15 09:36:03.830439
+./gradio_cached_examples/38/Generated image/9dbee7af-40cb-4416-ba6d-4a051fa8edb3,,,2023-06-15 09:36:10.699609
+./gradio_cached_examples/38/Generated image/148d3dde-4179-4292-b993-143fd15e9398,,,2023-06-15 09:36:14.053641
+./gradio_cached_examples/38/Generated image/60d193e4-3d27-4007-9120-6a96c6678db9,,,2023-06-15 09:36:19.154572

gradio_cached_examples/49/Generated image/94f49dad-28e0-4a46-b451-7b58f47bffe1/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/94f49dad-28e0-4a46-b451-7b58f47bffe1/5c0486001a1f02f907fa23894f6c74dfcd28bef1/image.png": null}

gradio_cached_examples/49/Generated image/9e75c543-0f51-4ed7-bc34-a80618c13c95/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/9e75c543-0f51-4ed7-bc34-a80618c13c95/3a25deedf57bac4c518f26ace13b04b794d086d9/image.png": null}

gradio_cached_examples/49/Generated image/a74de25c-de7c-4395-b5e2-f3d80d2e529e/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/a74de25c-de7c-4395-b5e2-f3d80d2e529e/6c40796796748fc1d9d3d73c184baf9c457ef147/image.png": null}

gradio_cached_examples/49/Generated image/d67589f2-e7e8-4a2d-be3c-3418f192e005/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/d67589f2-e7e8-4a2d-be3c-3418f192e005/bca7d034cf794d0f7f2b113bad1749c758090183/image.png": null}

gradio_cached_examples/49/Generated image/ec9faa2f-c428-4c04-be72-c9bb7ea91773/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/ec9faa2f-c428-4c04-be72-c9bb7ea91773/4ba2288b3416336f5f0830d6e74b9ab304010275/image.png": null}

gradio_cached_examples/49/log.csv

@@ -1,6 +0,0 @@
-Generated image,flag,username,timestamp
-./gradio_cached_examples/49/Generated image/ec9faa2f-c428-4c04-be72-c9bb7ea91773,,,2023-06-13 20:35:59.970601
-./gradio_cached_examples/49/Generated image/9e75c543-0f51-4ed7-bc34-a80618c13c95,,,2023-06-13 20:36:01.344415
-./gradio_cached_examples/49/Generated image/a74de25c-de7c-4395-b5e2-f3d80d2e529e,,,2023-06-13 20:36:02.741091
-./gradio_cached_examples/49/Generated image/94f49dad-28e0-4a46-b451-7b58f47bffe1,,,2023-06-13 20:36:04.072746
-./gradio_cached_examples/49/Generated image/d67589f2-e7e8-4a2d-be3c-3418f192e005,,,2023-06-13 20:36:05.445782

gradio_cached_examples/50/Generated image/3a792303-91fd-4d02-be6a-0575c8f98860/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/3a792303-91fd-4d02-be6a-0575c8f98860/81be364efed834f4d9789547319059758af80c17/image.png": null}

gradio_cached_examples/50/Generated image/737d9e8d-aa0b-40eb-9b4c-1c0a5d0dd0a5/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/737d9e8d-aa0b-40eb-9b4c-1c0a5d0dd0a5/f52a03b1385c26cc9f61a584fd1706b2e8079d6c/image.png": null}

gradio_cached_examples/50/Generated image/9b3fe1f3-7d57-47ac-b374-1e7c143557f8/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/9b3fe1f3-7d57-47ac-b374-1e7c143557f8/93b938a7b0d3661ae7003ff78d76fe134b78e5ab/image.png": null}

gradio_cached_examples/50/Generated image/b1a46f13-e06b-4fd2-ba14-eb14847c6c83/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/b1a46f13-e06b-4fd2-ba14-eb14847c6c83/af0d2cb9b5cd66079367c484fb9c22d2f00df744/image.png": null}

gradio_cached_examples/50/Generated image/d72b94e5-1656-4450-8758-8b2445d7e1ea/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/d72b94e5-1656-4450-8758-8b2445d7e1ea/678005178ce187762f93d2c1647556f15c5e74b6/image.png": null}

gradio_cached_examples/50/log.csv

@@ -0,0 +1,6 @@
+Generated image,flag,username,timestamp
+./gradio_cached_examples/50/Generated image/737d9e8d-aa0b-40eb-9b4c-1c0a5d0dd0a5,,,2023-06-15 09:36:20.512311
+./gradio_cached_examples/50/Generated image/9b3fe1f3-7d57-47ac-b374-1e7c143557f8,,,2023-06-15 09:36:21.885861
+./gradio_cached_examples/50/Generated image/b1a46f13-e06b-4fd2-ba14-eb14847c6c83,,,2023-06-15 09:36:23.281078
+./gradio_cached_examples/50/Generated image/d72b94e5-1656-4450-8758-8b2445d7e1ea,,,2023-06-15 09:36:24.611777
+./gradio_cached_examples/50/Generated image/3a792303-91fd-4d02-be6a-0575c8f98860,,,2023-06-15 09:36:25.986015