diff --git a/app.py b/app.py index c6acd592e7b969a6aef4682bf960ba11fe7cb6e8..d251f3b28a7ceb5cdd6331d454c4664f23a9b8aa 100644 --- a/app.py +++ b/app.py @@ -22,7 +22,7 @@ cfg = setup(args) multi_classes = False -device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" +device = "cuda" if torch.cuda.is_available() else "cpu" Ours, preprocess = models.load("CS-ViT-B/16", device=device, cfg=cfg, train_bool=False) state_dict = torch.load("sketch_seg_best_miou.pth", map_location=device) @@ -37,6 +37,9 @@ print("Model loaded successfully") def run(sketch, caption, threshold, seed): + # select a random seed between 1 and 10 for the color + color_seed = np.random.randint(0, 9) + # set the condidate classes here caption = caption.replace('\n',' ') translator = str.maketrans('', '', string.punctuation) @@ -49,7 +52,7 @@ def run(sketch, caption, threshold, seed): # print(classes) colors = plt.get_cmap("Set1").colors - classes_colors = colors[:len(classes)] + classes_colors = colors[color_seed:len(classes)+color_seed] sketch2 = sketch['composite'] @@ -127,9 +130,11 @@ def run(sketch, caption, threshold, seed): plt.imshow(rgb_image) plt.axis('off') plt.tight_layout() + # plt.savefig(f'poster_vis/{classes[0]}.png', bbox_inches='tight', pad_inches=0) plt.savefig('output.png', bbox_inches='tight', pad_inches=0) plt.close() + # rgb_image = Image.open(f'poster_vis/{classes[0]}.png') rgb_image = Image.open('output.png') return rgb_image @@ -212,9 +217,7 @@ async () => { element.textContent = firstWord; }); - document.querySelector('input[type=number]').disabled = true; - - + document.querySelector('input[type=number]').disabled = true; } """ @@ -401,13 +404,16 @@ with gr.Blocks(js=scripts, css=css, theme='gstaff/xkcd') as demo: with gr.Row(): with gr.Column(): # in_image = gr.Image( label="Sketch", type="pil", sources="upload" , height=512 ) - in_canvas_image = gr.Sketchpad( brush=gr.Brush(colors=["#000000"], color_mode="fixed" , default_size=2), + in_canvas_image = gr.Sketchpad( + # value=Image.new('RGB', (512, 512), color=(255, 255, 255)), + brush=gr.Brush(colors=["#000000"], color_mode="fixed" , default_size=2), image_mode="RGBA",elem_classes=["white-bg", "gray-border" , "border-radius" ,"own-shadow" ] , - label="Sketch" , canvas_size=(512,512) , sources=['upload'], - interactive=True , layers= False, transforms=[] ) + label="Sketch" , canvas_size=(512,512) ,sources=['upload'], + interactive=True , layers= False, transforms=[] + ) query_selector = 'button[aria-label="Upload button"]' + # with gr.Row(): - # segment_btn.click(fn=run, inputs=[in_image, in_textbox, in_slider], outputs=[out_image]) upload_draw_btn = gr.HTML(f"""
@@ -441,7 +447,7 @@ with gr.Blocks(js=scripts, css=css, theme='gstaff/xkcd') as demo: value=0.5 , interactive=True, step=0.05, minimum=0, maximum=1) with gr.Row(): - segment_btn = gr.Button( 'Segment it !' , elem_classes=["white-bg", "gray-border" , "border-radius" ,"own-shadow" , 'bold' , 'mt-mb-1' ] , size="sm") + segment_btn = gr.Button( 'Segment it* !' , elem_classes=["white-bg", "gray-border" , "border-radius" ,"own-shadow" , 'bold' , 'mt-mb-1' ] , size="sm") segment_btn.click(fn=run, inputs=[in_canvas_image , in_textbox , in_slider ], outputs=[out_image]) gallery_label = gr.HTML("

Gallery: you can click on any of the example sketches below to start segmenting them (or even drawing over them)

") @@ -498,5 +504,7 @@ with gr.Blocks(js=scripts, css=css, theme='gstaff/xkcd') as demo: fn=run, # cache_examples=True, ) - + + gr.HTML("
*For optimal performance, use a commercial Nvidia RTX 3090 (this demo runs on a basic 2 vCPU).
") + gr.HTML("
*We compare the entire caption to the scene sketch and threshold most similar pixels, without extracting individual classes.
") demo.launch(share=False) diff --git a/output.png b/output.png index 2eefdb654dcfc181b42787ce7b60b160ca95ca6f..9045131c20269960da7b6a150fb6cb4080849582 100644 Binary files a/output.png and b/output.png differ diff --git a/poster_vis/bench.png b/poster_vis/bench.png new file mode 100644 index 0000000000000000000000000000000000000000..7d7825cd895cf7322bb34f9bcdba03099dec8661 Binary files /dev/null and b/poster_vis/bench.png differ diff --git a/poster_vis/bench1.png b/poster_vis/bench1.png new file mode 100644 index 0000000000000000000000000000000000000000..421cb4fdea5a686f83533b8f47ae8b8014e1a0e9 Binary files /dev/null and b/poster_vis/bench1.png differ diff --git a/poster_vis/bird.png b/poster_vis/bird.png new file mode 100644 index 0000000000000000000000000000000000000000..3a5faad3318f9944ed410b874eee6c9d663fffeb Binary files /dev/null and b/poster_vis/bird.png differ diff --git a/poster_vis/branch.png b/poster_vis/branch.png new file mode 100644 index 0000000000000000000000000000000000000000..cc322dccbc47fa3aa99502e0a60dc91767965b4a Binary files /dev/null and b/poster_vis/branch.png differ diff --git a/poster_vis/car.png b/poster_vis/car.png new file mode 100644 index 0000000000000000000000000000000000000000..bb7d2082414da30a7a092ffd7fe0cf69c6f78de0 Binary files /dev/null and b/poster_vis/car.png differ diff --git a/poster_vis/chair.png b/poster_vis/chair.png new file mode 100644 index 0000000000000000000000000000000000000000..7e71292b87681d0e6f5dc667370365cb52f91c8c Binary files /dev/null and b/poster_vis/chair.png differ diff --git a/poster_vis/control tower.png b/poster_vis/control tower.png new file mode 100644 index 0000000000000000000000000000000000000000..be95e0f53993b27c914378ec83069f14963b43f4 Binary files /dev/null and b/poster_vis/control tower.png differ diff --git a/poster_vis/cow.png b/poster_vis/cow.png new file mode 100644 index 0000000000000000000000000000000000000000..f31a5788a103a97089d7d695ff192e9de23ad4fb Binary files /dev/null and b/poster_vis/cow.png differ diff --git a/poster_vis/demo/000000001611.png b/poster_vis/demo/000000001611.png new file mode 100644 index 0000000000000000000000000000000000000000..cd1c6ceb8712a384784671b517b747c998ce60af Binary files /dev/null and b/poster_vis/demo/000000001611.png differ diff --git a/poster_vis/demo/000000004068.png b/poster_vis/demo/000000004068.png new file mode 100644 index 0000000000000000000000000000000000000000..4d74ae0dd32695682c3ed4cb090f764595d40fcd Binary files /dev/null and b/poster_vis/demo/000000004068.png differ diff --git a/poster_vis/demo/000000004546.png b/poster_vis/demo/000000004546.png new file mode 100644 index 0000000000000000000000000000000000000000..b5d9a9c38c764f34536a7d8ef26f4f058de71a9f Binary files /dev/null and b/poster_vis/demo/000000004546.png differ diff --git a/poster_vis/demo/000000005076.png b/poster_vis/demo/000000005076.png new file mode 100644 index 0000000000000000000000000000000000000000..89b5637aa2c07107f76f538303e0cd1a4ad1fa7c Binary files /dev/null and b/poster_vis/demo/000000005076.png differ diff --git a/poster_vis/demo/000000006336.png b/poster_vis/demo/000000006336.png new file mode 100644 index 0000000000000000000000000000000000000000..4a2020464a4a2034ac2cfa5350de87e2241a00af Binary files /dev/null and b/poster_vis/demo/000000006336.png differ diff --git a/poster_vis/demo/000000011766.png b/poster_vis/demo/000000011766.png new file mode 100644 index 0000000000000000000000000000000000000000..5a2f76b0120eb617d3ef903207bfd1a44080b539 Binary files /dev/null and b/poster_vis/demo/000000011766.png differ diff --git a/poster_vis/demo/000000024458.png b/poster_vis/demo/000000024458.png new file mode 100644 index 0000000000000000000000000000000000000000..42e95279231bef67c9a24d477e4b047bce7e7ac5 Binary files /dev/null and b/poster_vis/demo/000000024458.png differ diff --git a/poster_vis/demo/000000024931.png b/poster_vis/demo/000000024931.png new file mode 100644 index 0000000000000000000000000000000000000000..1b626227ae22cb58feee6a9744ce316af52fabc7 Binary files /dev/null and b/poster_vis/demo/000000024931.png differ diff --git a/poster_vis/demo/000000034214.png b/poster_vis/demo/000000034214.png new file mode 100644 index 0000000000000000000000000000000000000000..c690f1c365b1f6b9f0e3123d7bfbd827a1f135dc Binary files /dev/null and b/poster_vis/demo/000000034214.png differ diff --git a/poster_vis/demo/000000038116.png b/poster_vis/demo/000000038116.png new file mode 100644 index 0000000000000000000000000000000000000000..87243ab4c074cc0c39ec3cc5e59419d3f807d7c9 Binary files /dev/null and b/poster_vis/demo/000000038116.png differ diff --git a/poster_vis/demo/000000045280.png b/poster_vis/demo/000000045280.png new file mode 100644 index 0000000000000000000000000000000000000000..8ee1a70c57cc4d3cc0c06f625a3846c34a073f50 Binary files /dev/null and b/poster_vis/demo/000000045280.png differ diff --git a/poster_vis/demo/000000221509.png b/poster_vis/demo/000000221509.png new file mode 100644 index 0000000000000000000000000000000000000000..c1fc9ae4664b908543579a8302dc02beda8bef25 Binary files /dev/null and b/poster_vis/demo/000000221509.png differ diff --git a/poster_vis/demo/000000246066.png b/poster_vis/demo/000000246066.png new file mode 100644 index 0000000000000000000000000000000000000000..bcb2cadf7bb3a92ea4fc2de93b296cb2d6f12517 Binary files /dev/null and b/poster_vis/demo/000000246066.png differ diff --git a/poster_vis/demo/000000260974.png b/poster_vis/demo/000000260974.png new file mode 100644 index 0000000000000000000000000000000000000000..a08f5e6814f8821df329d25b18f597025e62425f Binary files /dev/null and b/poster_vis/demo/000000260974.png differ diff --git a/poster_vis/demo/000000268340.png b/poster_vis/demo/000000268340.png new file mode 100644 index 0000000000000000000000000000000000000000..f99514b1fb965e02464442cb7145ecc7ea880121 Binary files /dev/null and b/poster_vis/demo/000000268340.png differ diff --git a/poster_vis/demo/000000305414.png b/poster_vis/demo/000000305414.png new file mode 100644 index 0000000000000000000000000000000000000000..6d4e903ed4aff81f8809f34b777024c9b50bc3aa Binary files /dev/null and b/poster_vis/demo/000000305414.png differ diff --git a/poster_vis/demo/000000406874.png b/poster_vis/demo/000000406874.png new file mode 100644 index 0000000000000000000000000000000000000000..8a60d17a4ee1a46f471e9dd80501e80e4307ad2f Binary files /dev/null and b/poster_vis/demo/000000406874.png differ diff --git a/poster_vis/demo/000000484246.png b/poster_vis/demo/000000484246.png new file mode 100644 index 0000000000000000000000000000000000000000..c518ddea6eb275f6c90b1f0f4c703e1f10d3644a Binary files /dev/null and b/poster_vis/demo/000000484246.png differ diff --git a/poster_vis/demo/000000549338.png b/poster_vis/demo/000000549338.png new file mode 100644 index 0000000000000000000000000000000000000000..ca852289c17e911233ecbb911de83027cb590314 Binary files /dev/null and b/poster_vis/demo/000000549338.png differ diff --git a/poster_vis/demo/sketch_1.png b/poster_vis/demo/sketch_1.png new file mode 100644 index 0000000000000000000000000000000000000000..bec02d2669669ab2e2d193ab9e3de26505d1f6ef Binary files /dev/null and b/poster_vis/demo/sketch_1.png differ diff --git a/poster_vis/demo/sketch_2.png b/poster_vis/demo/sketch_2.png new file mode 100644 index 0000000000000000000000000000000000000000..e94e25ef042c3827e4d791b2c4e4a5cac8e63250 Binary files /dev/null and b/poster_vis/demo/sketch_2.png differ diff --git a/poster_vis/demo/sketch_3.png b/poster_vis/demo/sketch_3.png new file mode 100644 index 0000000000000000000000000000000000000000..50ff3b0fb7638bf80be7bf69b8981a3c5f1a9d06 Binary files /dev/null and b/poster_vis/demo/sketch_3.png differ diff --git a/poster_vis/ears.png b/poster_vis/ears.png new file mode 100644 index 0000000000000000000000000000000000000000..41765a331b53f5e020a984015484d0ce9965870a Binary files /dev/null and b/poster_vis/ears.png differ diff --git a/poster_vis/elephant.png b/poster_vis/elephant.png new file mode 100644 index 0000000000000000000000000000000000000000..3bbf6941d54634044bd8fa3874398aee2cedae61 Binary files /dev/null and b/poster_vis/elephant.png differ diff --git a/poster_vis/giraffe.png b/poster_vis/giraffe.png new file mode 100644 index 0000000000000000000000000000000000000000..3063ea9958fa12d62a843d13e69a1cfa422db783 Binary files /dev/null and b/poster_vis/giraffe.png differ diff --git a/poster_vis/girrafe.png b/poster_vis/girrafe.png new file mode 100644 index 0000000000000000000000000000000000000000..0374c93d9728967585151a36507ece3e9beb8830 Binary files /dev/null and b/poster_vis/girrafe.png differ diff --git a/poster_vis/grass.png b/poster_vis/grass.png new file mode 100644 index 0000000000000000000000000000000000000000..7d06e8d6139e296bca25e0d81ab28f9c0562f8b8 Binary files /dev/null and b/poster_vis/grass.png differ diff --git a/poster_vis/hands.png b/poster_vis/hands.png new file mode 100644 index 0000000000000000000000000000000000000000..d1e8fb2ed81304bbb0fe8f1a8df0d84feba42225 Binary files /dev/null and b/poster_vis/hands.png differ diff --git a/poster_vis/hat.png b/poster_vis/hat.png new file mode 100644 index 0000000000000000000000000000000000000000..7db086d24382555791424ab7cadcb1914d2205df Binary files /dev/null and b/poster_vis/hat.png differ diff --git a/poster_vis/hat_1.png b/poster_vis/hat_1.png new file mode 100644 index 0000000000000000000000000000000000000000..c454dd6b48d7d547aa9120abee522f020943e6fb Binary files /dev/null and b/poster_vis/hat_1.png differ diff --git a/poster_vis/horse.png b/poster_vis/horse.png new file mode 100644 index 0000000000000000000000000000000000000000..22cbb72a816f9ab6672ce990085050d04abfb48a Binary files /dev/null and b/poster_vis/horse.png differ diff --git a/poster_vis/kite.png b/poster_vis/kite.png new file mode 100644 index 0000000000000000000000000000000000000000..ca5698606655cfee4fc9d6804cadfd301f13f1ff Binary files /dev/null and b/poster_vis/kite.png differ diff --git a/poster_vis/mountains1.png b/poster_vis/mountains1.png new file mode 100644 index 0000000000000000000000000000000000000000..ef4d324d06bc50af90e9fd18cc09e9d2dd7861aa Binary files /dev/null and b/poster_vis/mountains1.png differ diff --git a/poster_vis/mountains2.png b/poster_vis/mountains2.png new file mode 100644 index 0000000000000000000000000000000000000000..89e492b6a1d4892c013f0491403ab1c97b76348a Binary files /dev/null and b/poster_vis/mountains2.png differ diff --git a/poster_vis/mountains3.png b/poster_vis/mountains3.png new file mode 100644 index 0000000000000000000000000000000000000000..a77d900ded88370ff2c24a4a51523d02d42d89fc Binary files /dev/null and b/poster_vis/mountains3.png differ diff --git a/poster_vis/person.png b/poster_vis/person.png new file mode 100644 index 0000000000000000000000000000000000000000..efa1bc74e9770369495edc6a77414be1ea765047 Binary files /dev/null and b/poster_vis/person.png differ diff --git a/poster_vis/person1.png b/poster_vis/person1.png new file mode 100644 index 0000000000000000000000000000000000000000..e57c81342210cac6d68b5b1ca42ef7baf0d7d871 Binary files /dev/null and b/poster_vis/person1.png differ diff --git a/poster_vis/planes.png b/poster_vis/planes.png new file mode 100644 index 0000000000000000000000000000000000000000..02dbdfb80c8899fc887dba114b01194b26702cc3 Binary files /dev/null and b/poster_vis/planes.png differ diff --git a/poster_vis/pot.png b/poster_vis/pot.png new file mode 100644 index 0000000000000000000000000000000000000000..17e7d08ec16802fe0764adb213dfe92b166f7a82 Binary files /dev/null and b/poster_vis/pot.png differ diff --git a/poster_vis/sheeps.png b/poster_vis/sheeps.png new file mode 100644 index 0000000000000000000000000000000000000000..ff79a0a96786856323369d71c92b0a98e4da7688 Binary files /dev/null and b/poster_vis/sheeps.png differ diff --git a/poster_vis/skateboard.png b/poster_vis/skateboard.png new file mode 100644 index 0000000000000000000000000000000000000000..d91e4d683dca3ea14110c913d12abbb3ca8432d3 Binary files /dev/null and b/poster_vis/skateboard.png differ diff --git a/poster_vis/stop sign.png b/poster_vis/stop sign.png new file mode 100644 index 0000000000000000000000000000000000000000..775432ac78c9f4a8a833e2e195ea0177363abbc8 Binary files /dev/null and b/poster_vis/stop sign.png differ diff --git a/poster_vis/suit.png b/poster_vis/suit.png new file mode 100644 index 0000000000000000000000000000000000000000..59734c2bcc62ed0c9441e6e3ffb273ab28e877bf Binary files /dev/null and b/poster_vis/suit.png differ diff --git a/poster_vis/tree branch.png b/poster_vis/tree branch.png new file mode 100644 index 0000000000000000000000000000000000000000..9529652519c5356b4c23e28c8c31a0432ff9f315 Binary files /dev/null and b/poster_vis/tree branch.png differ diff --git a/poster_vis/tree2.png b/poster_vis/tree2.png new file mode 100644 index 0000000000000000000000000000000000000000..2fe4e4806215f8ffbc4d33244ec84c1f17cff582 Binary files /dev/null and b/poster_vis/tree2.png differ diff --git a/poster_vis/tree3.png b/poster_vis/tree3.png new file mode 100644 index 0000000000000000000000000000000000000000..0680d69595de51edb9803afb30a5b9eeda6003f5 Binary files /dev/null and b/poster_vis/tree3.png differ diff --git a/poster_vis/tree_1.png b/poster_vis/tree_1.png new file mode 100644 index 0000000000000000000000000000000000000000..5b3d787967410c91dde9cb5f14466bb55de7cb71 Binary files /dev/null and b/poster_vis/tree_1.png differ diff --git a/poster_vis/tree_5.png b/poster_vis/tree_5.png new file mode 100644 index 0000000000000000000000000000000000000000..4a1f7e2b9d7c9f71eddbba5f9e51014aaa8ae2a1 Binary files /dev/null and b/poster_vis/tree_5.png differ diff --git a/poster_vis/trees.png b/poster_vis/trees.png new file mode 100644 index 0000000000000000000000000000000000000000..eb9c398cbeee04347f1ee36ad356e12097c6cbea Binary files /dev/null and b/poster_vis/trees.png differ