fix UI for gradio upgrade

app.py

@@ -31,8 +31,14 @@ embedding_powers = [1. for i in range(max_tabs)]
 embedding_base64s = [None for i in range(max_tabs)]
 # embedding_base64s = gr.State(value=[None for i in range(max_tabs)])
 
+debug_print_on = False
+
+def debug_print(*args, **kwargs):
+    if debug_print_on:
+        print(*args, **kwargs)
 
 def image_to_embedding(input_im):
+    # debug_print("image_to_embedding")
     input_im = Image.fromarray(input_im)
     prepro = preprocess(input_im).unsqueeze(0).to(device)
     with torch.no_grad():
@@ -42,6 +48,7 @@ def image_to_embedding(input_im):
     return image_embeddings_np
 
 def prompt_to_embedding(prompt):
+    # debug_print("prompt_to_embedding")
     text = tokenizer([prompt]).to(device)
     with torch.no_grad():
         prompt_embededdings = model.encode_text(text)
@@ -50,6 +57,7 @@ def prompt_to_embedding(prompt):
     return prompt_embededdings_np
 
 def embedding_to_image(embeddings):
+    # debug_print("embedding_to_image")
     size = math.ceil(math.sqrt(embeddings.shape[0]))
     image_embeddings_square = np.pad(embeddings, (0, size**2 - embeddings.shape[0]), 'constant')
     image_embeddings_square.resize(size,size)
@@ -57,6 +65,7 @@ def embedding_to_image(embeddings):
     return embedding_image
 
 def embedding_to_base64(embeddings):
+    # debug_print("embedding_to_base64")
     import base64
     # ensure float32
     embeddings = embeddings.astype(np.float32)
@@ -64,12 +73,22 @@ def embedding_to_base64(embeddings):
     return embeddings_b64
 
 def base64_to_embedding(embeddings_b64):
+    # debug_print("base64_to_embedding")
     import base64
     embeddings = base64.urlsafe_b64decode(embeddings_b64)
     embeddings = np.frombuffer(embeddings, dtype=np.float32)
     # embeddings = torch.tensor(embeddings)
     return embeddings
 
+def is_prompt_embeddings(prompt):
+    if prompt is None or prompt == "":
+        return False
+    try:
+        embedding = base64_to_embedding(prompt)
+        return True
+    except Exception as e:
+        return False
+
 def safe_url(url):
     import urllib.parse
     url = urllib.parse.quote(url, safe=':/')
@@ -83,6 +102,7 @@ def main(
     embeddings,
     n_samples=4,
     ):
+    debug_print("main")
 
     embeddings = base64_to_embedding(embeddings)
     # convert to python array
@@ -117,17 +137,21 @@ def main(
     return images
 
 def on_image_load_update_embeddings(image_data):
+    debug_print("on_image_load_update_embeddings")
     # image to embeddings
     if image_data is None:
         # embeddings = prompt_to_embedding('')
         # embeddings_b64 = embedding_to_base64(embeddings)
         # return gr.Text.update(embeddings_b64)
-        return gr.Text.update('')
+        # return gr.Text.update('')
+        return ''
     embeddings = image_to_embedding(image_data)
     embeddings_b64 = embedding_to_base64(embeddings)
-    return gr.Text.update(embeddings_b64)
+    # return gr.Text.update(embeddings_b64)
+    return embeddings_b64
 
 def on_prompt_change_update_embeddings(prompt):
+    debug_print("on_prompt_change_update_embeddings")
     # prompt to embeddings
     if prompt is None or prompt == "":
         embeddings = prompt_to_embedding('')
@@ -135,9 +159,10 @@ def on_prompt_change_update_embeddings(prompt):
         return gr.Text.update(embedding_to_base64(embeddings))
     embeddings = prompt_to_embedding(prompt)
     embeddings_b64 = embedding_to_base64(embeddings)
-    return gr.Text.update(embeddings_b64)
+    return embeddings_b64
 
 def update_average_embeddings(embedding_base64s_state, embedding_powers):
+    debug_print("update_average_embeddings")
     final_embedding = None
     num_embeddings = 0
     for i, embedding_base64 in enumerate(embedding_base64s_state):
@@ -154,7 +179,7 @@ def update_average_embeddings(embedding_base64s_state, embedding_powers):
         # embeddings = prompt_to_embedding('')
         # embeddings_b64 = embedding_to_base64(embeddings)
         # return gr.Text.update(embeddings_b64)
-        return gr.Text.update('')
+        return ''
 
     # TODO toggle this to support average or sum
     # final_embedding = final_embedding / num_embeddings
@@ -166,22 +191,25 @@ def update_average_embeddings(embedding_base64s_state, embedding_powers):
     return embeddings_b64
 
 def on_power_change_update_average_embeddings(embedding_base64s_state, embedding_power_state, power, idx):
+    debug_print("on_power_change_update_average_embeddings")
     embedding_power_state[idx] = power
     embeddings_b64 = update_average_embeddings(embedding_base64s_state, embedding_power_state)
-    return gr.Text.update(embeddings_b64)
+    return embeddings_b64
 
 def on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embedding_base64, idx):
+    debug_print("on_embeddings_changed_update_average_embeddings")
     embedding_base64s_state[idx] = embedding_base64 if embedding_base64 != '' else None
     embeddings_b64 = update_average_embeddings(embedding_base64s_state, embedding_power_state)
-    return gr.Text.update(embeddings_b64)    
+    return embeddings_b64
 
 def on_embeddings_changed_update_plot(embeddings_b64):
+    debug_print("on_embeddings_changed_update_plot")
     # plot new embeddings
     if embeddings_b64 is None or embeddings_b64 == "":
         data = pd.DataFrame({
             'embedding': [],
             'index': []})        
-        return gr.LinePlot.update(data,
+        update = gr.LinePlot.update(data,
             x="index",
             y="embedding",
             # color="country",
@@ -192,6 +220,7 @@ def on_embeddings_changed_update_plot(embeddings_b64):
             # stroke_dash_legend_title="Country Cluster",
             # height=300,
             width=0)
+        return update
         
     embeddings = base64_to_embedding(embeddings_b64)
     data = pd.DataFrame({
@@ -210,6 +239,7 @@ def on_embeddings_changed_update_plot(embeddings_b64):
             width=embeddings.shape[0])
 
 def on_example_image_click_set_image(input_image, image_url):
+    debug_print("on_example_image_click_set_image")
     input_image.value = image_url
 
 # device = torch.device("mps" if torch.backends.mps.is_available() else "cuda:0" if torch.cuda.is_available() else "cpu")
@@ -236,7 +266,7 @@ examples = [
     # ["SohoJoeEth.jpeg", "Snoop Dogg.jpg", "SohoJoeEth + Snoop Dogg.jpeg"],
     ["pup1.jpg", "", "Pup no teacup.jpg"],
 ]
-tile_size = 100
+tile_size = 110
 # image_folder = os.path.join("file", "images")
 image_folder ="images"
 
@@ -349,7 +379,7 @@ Try uploading a few images and/or add some text prompts and search the embedding
                         # input_image.change(on_image_load, inputs= [input_image, plot])
                 with gr.Row():
                     with gr.Column(scale=2, min_width=240):
-                        input_prompts[i] = gr.Textbox(label="Text Prompt", show_label=True)
+                        input_prompts[i] = gr.Textbox(label="Text Prompt", show_label=True, max_lines=4)
                     with gr.Column(scale=3, min_width=600):
                         with gr.Row():
                             # with gr.Slider(min=-5, max=5, value=1, label="Power", show_label=True):
@@ -357,7 +387,7 @@ Try uploading a few images and/or add some text prompts and search the embedding
                             embedding_powers[i] = gr.Slider(minimum=-3, maximum=3, value=1, label="Power", show_label=True, interactive=True)
                         with gr.Row():
                             with gr.Accordion(f"Embeddings (base64)", open=False):
-                                embedding_base64s[i] = gr.Textbox(show_label=False)
+                                embedding_base64s[i] = gr.Textbox(show_label=False, live=True)
                 for idx, (tab_title, examples) in enumerate(tabbed_examples.items()):
                     with gr.Tab(tab_title):
                         with gr.Row():
@@ -395,15 +425,52 @@ Try uploading a few images and/or add some text prompts and search the embedding
 
     embedding_base64s_state = gr.State(value=[None for i in range(max_tabs)])
     embedding_power_state = gr.State(value=[1. for i in range(max_tabs)])
+
+    def on_image_load(input_image, idx_state, embedding_base64s_state, embedding_power_state):
+        debug_print("on_image_load")
+        embeddings_b64 = on_image_load_update_embeddings(input_image)
+        new_plot = on_embeddings_changed_update_plot(embeddings_b64)
+        average_embeddings_b64 = on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embeddings_b64, idx_state)
+        new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64)
+        return embeddings_b64, new_plot, average_embeddings_b64, new_average_plot
+
+    def on_prompt_change(prompt, idx_state, embedding_base64s_state, embedding_power_state):
+        debug_print("on_prompt_change")
+        if is_prompt_embeddings(prompt):
+            embeddings_b64 = prompt
+        else:
+            embeddings_b64 = on_prompt_change_update_embeddings(prompt)
+        new_plot = on_embeddings_changed_update_plot(embeddings_b64)
+        average_embeddings_b64 = on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embeddings_b64, idx_state)
+        new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64)
+        return embeddings_b64, new_plot, average_embeddings_b64, new_average_plot
+
+    def on_power_change(power, idx_state, embedding_base64s_state, embedding_power_state):
+        debug_print("on_power_change")
+        average_embeddings_b64 = on_power_change_update_average_embeddings(embedding_base64s_state, embedding_power_state, power, idx_state)
+        new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64)
+        return average_embeddings_b64, new_average_plot
+
     for i in range(max_tabs):
-        input_images[i].change(on_image_load_update_embeddings, input_images[i], [embedding_base64s[i]])
-        input_prompts[i].change(on_prompt_change_update_embeddings, input_prompts[i], [embedding_base64s[i]])
-        embedding_base64s[i].change(on_embeddings_changed_update_plot, embedding_base64s[i], [embedding_plots[i]])
         idx_state = gr.State(value=i)
-        embedding_base64s[i].change(on_embeddings_changed_update_average_embeddings, [embedding_base64s_state, embedding_power_state, embedding_base64s[i], idx_state], average_embedding_base64)
-        embedding_powers[i].change(on_power_change_update_average_embeddings, [embedding_base64s_state, embedding_power_state, embedding_powers[i], idx_state], average_embedding_base64)
+        input_images[i].change(on_image_load, 
+            [input_images[i], idx_state, embedding_base64s_state, embedding_power_state],
+            [embedding_base64s[i], embedding_plots[i], average_embedding_base64, average_embedding_plot])
+        input_prompts[i].change(on_prompt_change,
+            [input_prompts[i], idx_state, embedding_base64s_state, embedding_power_state],
+            [embedding_base64s[i], embedding_plots[i], average_embedding_base64, average_embedding_plot])
+        embedding_powers[i].change(on_power_change,
+            [embedding_powers[i], idx_state, embedding_base64s_state, embedding_power_state],
+            [average_embedding_base64, average_embedding_plot])
+
+
+        # input_images[i].change(on_image_load_update_embeddings, input_images[i], embedding_base64s[i])
+        # input_prompts[i].change(on_prompt_change_update_embeddings, input_prompts[i], embedding_base64s[i])
+        # embedding_base64s[i].change(on_embeddings_changed_update_plot, embedding_base64s[i], embedding_plots[i])
+        # embedding_base64s[i].change(on_embeddings_changed_update_average_embeddings, [embedding_base64s_state, embedding_power_state, embedding_base64s[i], idx_state], average_embedding_base64)
+        # embedding_powers[i].change(on_power_change_update_average_embeddings, [embedding_base64s_state, embedding_power_state, embedding_powers[i], idx_state], average_embedding_base64)
 
-    average_embedding_base64.change(on_embeddings_changed_update_plot, average_embedding_base64, average_embedding_plot)
+    # average_embedding_base64.change(on_embeddings_changed_update_plot, average_embedding_base64, average_embedding_plot)
 
     # submit.click(main, inputs= [embedding_base64s[0], scale, n_samples, steps, seed], outputs=output)
     submit.click(main, inputs= [average_embedding_base64, n_samples], outputs=output)
@@ -439,4 +506,4 @@ My interest is to use CLIP for image/video understanding (see [CLIP_visual-spati
 # ![Alt Text](file/pup1.jpg){height=100 width=100}    
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(debug=True)