import gradio as gr import torch from PIL import Image # from torchvision import transforms # from diffusers import StableDiffusionPipeline, StableDiffusionImageVariationPipeline, DiffusionPipeline import numpy as np import pandas as pd import math # from transformers import CLIPTextModel, CLIPTokenizer import os from api_helper import encode_numpy_array, decode_numpy_array, preprocess_image # clip_model_id = "openai/clip-vit-large-patch14-336" # clip_retrieval_indice_name, clip_model_id ="laion5B-L-14", "/laion/CLIP-ViT-L-14-laion2B-s32B-b82K" clip_retrieval_service_url = "https://knn.laion.ai/knn-service" # available models = ['RN50', 'RN101', 'RN50x4', 'RN50x16', 'RN50x64', 'ViT-B/32', 'ViT-B/16', 'ViT-L/14', 'ViT-L/14@336px'] # clip_model="ViT-B/32" clip_model="ViT-L/14" clip_image_size = 224 clip_model_id ="laion5B-L-14" max_tabs = 10 input_images = [None for i in range(max_tabs)] input_prompts = [None for i in range(max_tabs)] embedding_plots = [None for i in range(max_tabs)] embedding_powers = [1. for i in range(max_tabs)] # global embedding_base64s embedding_base64s = [None for i in range(max_tabs)] # embedding_base64s = gr.State(value=[None for i in range(max_tabs)]) debug_print_on = False def debug_print(*args, **kwargs): if debug_print_on: print(*args, **kwargs) def image_to_embedding(input_im): # debug_print("image_to_embedding") input_im = Image.fromarray(input_im) prepro = preprocess(input_im).unsqueeze(0).to(device) with torch.no_grad(): image_embeddings = model.encode_image(prepro) image_embeddings /= image_embeddings.norm(dim=-1, keepdim=True) image_embeddings_np = image_embeddings.cpu().to(torch.float32).detach().numpy() return image_embeddings_np def prompt_to_embedding(prompt): # debug_print("prompt_to_embedding") text = tokenizer([prompt]).to(device) with torch.no_grad(): prompt_embededdings = model.encode_text(text) prompt_embededdings /= prompt_embededdings.norm(dim=-1, keepdim=True) prompt_embededdings_np = prompt_embededdings.cpu().to(torch.float32).detach().numpy() return prompt_embededdings_np def embedding_to_image(embeddings): # debug_print("embedding_to_image") size = math.ceil(math.sqrt(embeddings.shape[0])) image_embeddings_square = np.pad(embeddings, (0, size**2 - embeddings.shape[0]), 'constant') image_embeddings_square.resize(size,size) embedding_image = Image.fromarray(image_embeddings_square, mode="L") return embedding_image def embedding_to_base64(embeddings): # debug_print("embedding_to_base64") import base64 # ensure float32 embeddings = embeddings.astype(np.float32) embeddings_b64 = base64.urlsafe_b64encode(embeddings).decode() return embeddings_b64 def base64_to_embedding(embeddings_b64): # debug_print("base64_to_embedding") import base64 embeddings = base64.urlsafe_b64decode(embeddings_b64) embeddings = np.frombuffer(embeddings, dtype=np.float32) # embeddings = torch.tensor(embeddings) return embeddings def is_prompt_embeddings(prompt): if prompt is None or prompt == "": return False try: embedding = base64_to_embedding(prompt) return True except Exception as e: return False def is_prompt_in_example_embeddings(prompt): keys = example_embeddings.keys() if prompt is None or prompt == "": return False if prompt in keys: return True return False def safe_url(url): import urllib.parse url = urllib.parse.quote(url, safe=':/') # if url has two .jpg filenames, take the first one if url.count('.jpg') > 0: url = url.split('.jpg')[0] + '.jpg' return url def main( # input_im, embeddings, n_samples=4, ): debug_print("main") # images = [] # for url in test_images_urls: # import requests # from io import BytesIO # from PIL import Image # try: # response = requests.get(url) # if not response.ok: # continue # bytes = BytesIO(response.content) # image = Image.open(bytes) # if image.mode != 'RGB': # image = image.convert('RGB') # # width = 336 # # aspect_ratio = float(image.height) / float(image.width) # # height = int(width * aspect_ratio) # # image = image.resize((width, height), Image.Resampling.LANCZOS) # images.append((image, "title")) # except Exception as e: # print(e) # return images embeddings = base64_to_embedding(embeddings) # convert to python array embeddings = embeddings.tolist() results = clip_retrieval_client.query(embedding_input=embeddings) images = [] for result in results: if len(images) >= n_samples: break url = safe_url(result["url"]) similarty = float("{:.4f}".format(result["similarity"])) title = str(similarty) + ' ' + result["caption"] # we could just return the url and the control would take care of the rest # however, if the url returns an error, the page crashes. # images.append((url, title)) # continue # dowload image import requests from io import BytesIO try: response = requests.get(url) if not response.ok: continue bytes = BytesIO(response.content) image = Image.open(bytes) if image.mode != 'RGB': image = image.convert('RGB') # width = 336 # aspect_ratio = float(image.height) / float(image.width) # height = int(width * aspect_ratio) # image = image.resize((width, height), Image.Resampling.LANCZOS) images.append((image, title)) except Exception as e: print(e) return images def on_image_load_update_embeddings(image_data): debug_print("on_image_load_update_embeddings") # image to embeddings if image_data is None: # embeddings = prompt_to_embedding('') # embeddings_b64 = embedding_to_base64(embeddings) # return gr.Text.update(embeddings_b64) # return gr.Text.update('') return '' embeddings = image_to_embedding(image_data) embeddings_b64 = embedding_to_base64(embeddings) # return gr.Text.update(embeddings_b64) return embeddings_b64 def on_image_as_payload_update_embeddings(payload): debug_print("on_image_as_payload_update_embeddings") if payload is None or payload == "": return '' image_data = decode_numpy_array(payload) embeddings = image_to_embedding(image_data) embeddings_b64 = embedding_to_base64(embeddings) # return gr.Text.update(embeddings_b64) return embeddings_b64 def on_prompt_change_update_embeddings(prompt): debug_print("on_prompt_change_update_embeddings") # prompt to embeddings if prompt is None or prompt == "": embeddings = prompt_to_embedding('') embeddings_b64 = embedding_to_base64(embeddings) return gr.Text.update(embedding_to_base64(embeddings)) embeddings = prompt_to_embedding(prompt) embeddings_b64 = embedding_to_base64(embeddings) return embeddings_b64 def update_average_embeddings(embedding_base64s_state, embedding_powers): debug_print("update_average_embeddings") final_embedding = None num_embeddings = 0 for i, embedding_base64 in enumerate(embedding_base64s_state): if embedding_base64 is None or embedding_base64 == "": continue embedding = base64_to_embedding(embedding_base64) embedding = embedding * embedding_powers[i] if final_embedding is None: final_embedding = embedding else: final_embedding = final_embedding + embedding num_embeddings += 1 if final_embedding is None: # embeddings = prompt_to_embedding('') # embeddings_b64 = embedding_to_base64(embeddings) # return gr.Text.update(embeddings_b64) return '' # TODO toggle this to support average or sum # final_embedding = final_embedding / num_embeddings # normalize embeddings in numpy final_embedding /= np.linalg.norm(final_embedding) embeddings_b64 = embedding_to_base64(final_embedding) return embeddings_b64 def on_power_change_update_average_embeddings(embedding_base64s_state, embedding_power_state, power, idx): debug_print("on_power_change_update_average_embeddings") embedding_power_state[idx] = power embeddings_b64 = update_average_embeddings(embedding_base64s_state, embedding_power_state) return embeddings_b64 def on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embedding_base64, idx): debug_print("on_embeddings_changed_update_average_embeddings") embedding_base64s_state[idx] = embedding_base64 if embedding_base64 != '' else None embeddings_b64 = update_average_embeddings(embedding_base64s_state, embedding_power_state) return embeddings_b64 def on_embeddings_changed_update_plot(embeddings_b64): debug_print("on_embeddings_changed_update_plot") # plot new embeddings if embeddings_b64 is None or embeddings_b64 == "": data = pd.DataFrame({ 'embedding': [], 'index': []}) update = gr.LinePlot.update(data, x="index", y="embedding", # color="country", title="Embeddings", # stroke_dash="cluster", # x_lim=[1950, 2010], tooltip=['index', 'embedding'], # stroke_dash_legend_title="Country Cluster", # height=300, width=0) return update embeddings = base64_to_embedding(embeddings_b64) data = pd.DataFrame({ 'embedding': embeddings, 'index': [n for n in range(len(embeddings))]}) return gr.LinePlot.update(data, x="index", y="embedding", # color="country", title="Embeddings", # stroke_dash="cluster", # x_lim=[1950, 2010], tooltip=['index', 'embedding'], # stroke_dash_legend_title="Country Cluster", # height=300, width=embeddings.shape[0]) def on_example_image_click_set_image(input_image, image_url): debug_print("on_example_image_click_set_image") input_image.value = image_url # device = torch.device("mps" if torch.backends.mps.is_available() else "cuda:0" if torch.cuda.is_available() else "cpu") device = "cuda:0" if torch.cuda.is_available() else "cpu" from clip_retrieval.load_clip import load_clip, get_tokenizer from clip_retrieval.clip_client import ClipClient, Modality model, preprocess = load_clip(clip_model, use_jit=True, device=device) tokenizer = get_tokenizer(clip_model) clip_retrieval_client = ClipClient( url=clip_retrieval_service_url, indice_name=clip_model_id, use_safety_model = False, use_violence_detector = False, # modality = Modality.TEXT, ) # model, preprocess, tokenizer, clip_retrieval_client = None, None, None, None examples = [ # ["SohoJoeEth.jpeg", "Ray-Liotta-Goodfellas.jpg", "SohoJoeEth + Ray.jpeg"], # ["SohoJoeEth.jpeg", "Donkey.jpg", "SohoJoeEth + Donkey.jpeg"], # ["SohoJoeEth.jpeg", "Snoop Dogg.jpg", "SohoJoeEth + Snoop Dogg.jpeg"], ["pup1.jpg", "", "Pup no teacup.jpg"], ] # image_folder = os.path.join("file", "images") image_folder ="images" # image_examples = { # "452650": "452650.jpeg", # "Prompt 1": "a college dorm with a desk and bunk beds", # "371739": "371739.jpeg", # "Prompt 2": "a large banana is placed before a stuffed monkey.", # "557922": "557922.jpeg", # "Prompt 3": "a person sitting on a bench using a cell phone", # } test_images_urls = [ "https://www.mdig.com.br/imagens/bichos/caes_miniatura_chicara_07.jpg", "https://i.pinimg.com/236x/04/ac/0e/04ac0e05964b75c9db59de94c571339e.jpg", "https://i.pinimg.com/236x/a1/55/9e/a1559e56ae5fb6e4d19c43c6396b5940--teacup-yorkie-yorkie-puppy.jpg", "https://i.pinimg.com/236x/11/c4/51/11c4518febb0869bd2b2d391d0753f44.jpg", ] example_embeddings = { "#Black & White embeddings": "F0kxPHAqE7t3DoY79djWOwA6Cb2hjK88EkuIvXdEgTzS2yY93WXvOsKffL08qjU9oGVJvZtXD7wiQ-u7QTLhvGRqozpSFqo8fCMaOy42NDyyXCC9ls69Olk_A7zJ6Ik97AwLOyNjCryYr4W8kREmPfIOPb0xrde7137Fu3Jr5bwGKGU90T-lvI1pMT1ftz-9qy3vPMTnRDzx97C8fRWjPGbQU71d6f26ASZyPdg3Qrx-saS9FaAbu83DK732Ry-9WQ7HPPPiwTzY4gS97gc1PGXmRrzsZUS9kQwmPKDZvzw4F4a9zElPPQjmdj2Lqak9SHFXPJmPvDwRLPU7YvqHPV7OYDx7K-q8wfdIveWEXT1kYE289sfOOwH6YbuID129kMivu1uvCb35jGi9shisPNsXz7zb0xk99u_ivE68QL2ibjo8jCAmvPIz5Lyv9kU8rUIKPbCciD23RQG9P48tPfEpsLwbZkE8wtjHvHqvB72k_Au818IRO2pLHz2U2yq9M1_hvTG2FD05si275m4rvL85ojtDSCo9xWclvTOEgLrMt449xggSvFQzT72wtOw7mtgLvRe-N732MlW8bGIWvMi_m7p1XSA91Sq2PHDDxrxcMC29RPoJPVeIzTyCC448PVequz8rLz2Rsnk6yayDvVjAJjuJxqe9ivRpvVEKjTwwxoM9OKOtPUjlhT20uYO8ynh2PYLVZjwREFS8wWWIPLZGrz2oNxy7GR4gvaENaDzBxJu8ZdYGtw7B7rxG2sC7mP0QPYkKXbzbh4E7HrsqvJ64P73AL_Y8I6L-vB59jjyVE8O7jshZOw19JLwQD1A9NG3xu00JHr08iBC9WANYvBe-N73tpoS6dOZaPT1Btzy9kcI8MzNRvNH7hDuMnwK8o88MvZWU5rwDUhk9sBEPPGi_DL0BY8u8-cKPvGqIjbw90_c8y19CvVYT8bxLLxU920igvX_DgTyAzh-8q2lZOy-UmbxXnlU8or9cPJruuj3HJHy9a5-ZPUJ717w9frq88QG2vIwxQz2dIam8ET5Uu2ftYzsxRxI9nDwcPDQz5jxtrf079M5rPAVIrryU_U87fslzPSTkQrxpCK08VY1NvdsXTzuwsz48rvvavIsTY705S2G936llvTsk0bwQhEG6zoYovD0TXjoseXq9bt45PfQJqDzYp_I8z3WLvGTflLwWLui78YjDvAx4Trw2-yO9oXunvADWCj1wlQQ8NmF-PG1uszzCusW8jujMO3W637yShqy7O4loPHI4ybw4yr271thYPB9hAj0PwUQ8L0fRPLkzlD091oY88PvBOjv4wDyBMh69eUsePfOb6DyI4Zq8TxH-u0uCi702EZc8kqoQvQQkAbwz9AY841FFvebosTs2z708FpwnPUqOPb0zlQi9KNeWvBbeALwy70U91KD_PHPxSTwi8wO8aroMPYpCdb1QLDI9VgwAvL-JKz1I1Za8EshhO-EUbD2cpQW9UEeQvA8vBD0KvYq6ZJ6DvWZsVb1UdT07gHVPPUM-VLxbYZM8SoXXPFP6iDwC-AU9_5p4vIuPKD0d7x26nrBSvWaCxjn2aCY9iaLDvD8yYbzQPzo9BihlPaxUar2IeMa7zJniOsRbiD0zigS9P-TqvJMO7bsuR3u8KS_WvEL1XT3vWoG7OmGqvBsJdb3_vlE5js_2vP-wF70sbJq8XqWJPOOOM71wkJk9WJRqvA7XNzqVV-O8OjBuPW5_O71u-QK92baeOovezDw-NPS8G-dkvRx_UD1xzYe94dxoPM0_-TzqrSE8DUWhPLz6qzwr4TW9yOPIO5FmYzyM_oC9fj8gvWtdljuotQg8lwlDPPPRubw4Bxc8f6y5u_x2trxXH2Q81m2TPadef7yfNwc9AlIuvKNWXr2A2c28s6e1uspCsrzBHtk66N90PeUS-TqID109vydpu8vDVb1ZknC97JqRu18mLT1hIZg8CNmBt43v1DsezNw73krSPH_eo7yAPlA99-k0PNBamLw16jA90kSlu-pTIz0DPLs8sdQLvMP3Mz3_zCS6UOMmvX4mFr3OhtK7tjA8PT6wJL2DTy69bE17vCFNVr1LJVS8zIuouVlRzjy34jA9CV-QPD5j3DwrYu68CD4uvZAP57zhOQs8krApvOvEV707mAC7giNdPYjSbjxfdLg8gZacu3ritLz6NPI7KDTBvEEhxDzihwE8pX0vPWS4GTtZfk28YPBwPQMaFr3Jqxs8t5bsuw3moj0q2q07AuRuPDSLnbw1-zi961XVuq_qKL1ofAW8CwGBvVScoz3uDny9px98PA35cTxwaYk6jwUJPQUyZbx2mVM9SMqnPdPMejxuREA9dK0UPe4VCLrn6vg8LBTOO8c2v7yL1wW9PUWJPIqrybzN3h67_-sSvVaMHz2DKno8uDCSPN3ZiL2B5JK9-CsOPNdBrbzEIPY7gCjdvEAJCjv3LX-8NjupvTIxijw55-I7CgjIvOx1Hj33TUg81is6vLs8Gj1K7VA8nyetPXnXAbgJ1US92c7tPNAYP7zzeVi9eaVGvaT8izv9_pi92S3sPCywZD3ihPK8pJ2NPZe0BT3EYSG9ahv8uic4k7wNc-M8bX9QvTUsirzUYaC8LCVrPS5Y7jyqsnk4UYbaPLF8P70_x7C80qB_u5jBjDzQqCO9vqn8vCUcMb3Z6TY9ZLRcPYLJHz3ZPJg7OMW9O23YBr3YeZu89uKCPTdrqjuwB-O8ZL82vJ7ClbvHOq-76d4xvITNT7z2x6Q8NQAkvatTUb2PhlY8NCe0OzOLR71EliA8-W6APBeomjstTFG99YfouysDNT2R3b28y7K4vLt2-bvKf6C9bf6XvH8EBrzZMak8BNpuPLLJhz1G6gU9UouwPE-QRT1lKKA8bjaGO9JQGDwpjtQ7yi_BvLv_ATyE1Ju99_8nPRh1gr1Vyyk6Q_WzPc60fz2-XoC9nsTxvJfFDT2zS9m8kx-2u3mlxjyfqf68Tje9PcK_G7v_F467sdXOvIzZdr3LONw8ngu2vF8Rgr35KP87TzCEvQNZS7xGrkU976MhPTOLsr2woYg7OGAmvRPIzLxLtaM7ofWDPXREq7wSwsi7T7dVPEiO5zwaOtu8OudNvHa5nLxYLpC9h445OydP3rsAwMG7rV9ZvWnBfb0Gwoo8ZwPXvPa72ztudDc9l4gfPJTxsjxCioM90JIwPaFs5rxs46q8yGKQux4injyRLZ281PZivejwfLwwZGE8NTinvDfsdzwSkMk8hMtmPSTkrbyigau8L4kVvJntIrrwqTo8dNSPvctfQruCT0M8lm1Wu6mpPD2Mmpc971YEOY4sGT2Be1M9rewmPfCeS7yGn9Y8KXYaPVbF0DvtdQm9r2bhvPwGGz1jNOe8KQ0xPbEPOz38nIO8PEQaPKFsZj30PNW9Be4FvRVfzj2HUvq7aJiRPB-lNz1GIze87PZrPKefPz3g_ni8VfOPvYRFbb1j-7W8_Z0xvVLvrrz8kZQ9_2dLPbAC4zxthKa8fzwJvff_p7waPzG9FV_jvLKlDr3ED249QDy3PD1SabxxX0i9M0hAvVL1Rz0cGCG9coYuvY--bj1sKhO9d_jmPHowQL0Bm_g863DIPUM-VD0OVhQ8I5H2PLwD_rxb5zY7MJqdOy6wOrxhIS09yOU7PT74ljwRi_M9WYDLvAtKIb01Yf68WfwQPf2YsTv0-vu8bp_qPHKDrL1z1KS8Xf_bPDCQXD1oIZE9lGFjuzaN5LzWIXk8nJsvPSgdtTyqEEq9286EOyZm6rtF8DM95ugxugOlDz115YI86aumO388CT1SPRC8MtM5vB6liDwqxT06Q5agvJ-TAz0CikY9MxbBvP2_QTwxhsc8NLeDOzql9LxJxmq8MSWCPfVtkTzuo3W78-gSvenIvrxOofe8zr5APQBnCL3z4us7cKEhvQ4lWL0FTwK9cMMcPHpNZT3G1hK9XOKhvZF2nz1mRdq8dgeou_fTQTz43968VndaPNdZ_DsoxLo8n0gPvTcp0Tq17Js85EnDPWiyZ733Fsk8AFwuPGuUv7sYHcs8GGtBvZK7dju76bi9KHsavWsqEzwk0xA8", "#Fire embeddings": "VsgiPPjckjuCP-q8fs25PC1DT7z-1li7_bAkvPkCrDx4W9E7V9sYvf3_D72Oyau8vXGFPVCSFD00qhO9-mSEPGQ8kT1_pN67qvXxvDwZDb1nLnO8TbvSO8Gn0j3OdUc9dNaqPP2wJL170AQ9CX-0PJu1jTycyBW9JAVvvAtWPjyF1pe8o4awO12vBrvkJ0O8EtsBPUszvbzduJO8zghmPeIBezyBaKE8STaIPBUnkDq7D1q8swJCPUR1kLy9qmU7itcaPZ2flr2fOlg7gcpKvQh__DzNAEM8xqSJPVNTb7s7aMA7cJ3sPK8ujDx4W1G8-RVhvcra-jxRuD-9RTkAPdr0dDx9a7Q7Y1q9ux2WLT0QBJO8DWkHPUE8pby9Ioi8kFydPJX0Xbzwpme8YeVuvL0iiD0q2Ti9oE1OPGXPlL0qswS805g7vWb1pD1ivPi8oOskvAm7BT323P685LIju0N1WL0WYLq80sEWPShRx7zTSVC83qLkvF-_1bwZ1RG9833EvIRlVr0jo2m8xxmpvGXirjyLTDo9hRa-PKAnGj2tVwK9aGogvay5xjxvZB69MoSnPHSHmzxcc1k79hhQvfIufbyH7Zo9E2NNvRq_0Dz0o_i8Tx2iPJb0lTypgPY8g90cPV7VljuyyZc7gXvfPLjW97x5W6097CGdvJhT9Lk7yum8AJsbPP90izw3uj49qUeEvQE277xqQaE70UybPIw2-Tq1O2y8yD9dPD0_wbxZY6485v7DvN3LP7vsg8Y8OZE_PehzGzwNj1-86kouPW2Ni70FM9w7wDIzvHUibzwqO_Q7c4djvc2eh73TIwq9icSkvA7x0rzb9Ky9Lpp4vaSs0rwUdkM6WLK9PFhQOL2vZ348TUYPveAqVrw3HOi87JbyPPGmXj06foG6E1CPPMgD-rx7gT29kgpMPau5fLxaTe271DaSvUjUpr0v6a28hLRBvS-tbjznpHy8rbm0uic-UTwCwZm8UOEavCZnmrz9Ek69cU65vcUGabznc0g8oZwDPWVX8ry3YVi9MPwRPZqilzu_H-E8m7WNOs-INLy-5pI85ZzivLewQ7wlVLY9WnaAPOMBMz1j-JM8MF47Pfe2gjuxjWq5LWEqPRohejwumni84ow3vBoOvLzHyqu8Z7mvvB1t9jzE83I83LglvQRv4zxAx448KRUuvI3UGT0Ojxe9IJM-O-fCzjwKVv29DMtUPX6R6LwDIwQ9gwNRPaOG1LzCSsA8f20AvbUoLj1SyzU9o3MNPJd8cz0mBQO8A0kvPVhQFDx-axC9Ts7IPEtkxDyxjeq6O7fPPI1Jbz3VDS49UQcrOsPg_LwpUSM8VwFxve7P5rwOQCw61uTbu1WMdbxV22A8IM-PPKsxHz1U8SG9cmGvvD1Kgjx3cRI9WMVXvPigwbxR9JA9zoj8vJJGlDzWlXC9N7q-vFGlAT0Ywps6XoYrvD946zy3_668ZZNnu4OhSz3WqHi9dl4cuE1ZKb3fj4I82s4KvILKJr3hjFu8w35TPZDker20d2E9DS22PRBmmLxmaug8D_EKPRUnEDsbcB2911n7u0PEwzyl-yu9lWl9vFFWljzP_Xc7i5slvU5_yzzJtGo80RBKvcKUFLxQkgs8jXImvDUyRLxjWr07RxCcPfPfybw_eGu8HZaJu9VvPDzCIwK8XjecPC9LRT56bkc9tWSRvFD0Rr2QgtE77x7SvPbUkzyikR48vVv6PPSQujuS96i9G10Du50wKj0UdsO5NIFcvGxnjbtLRnu9ZvVIvDI1vLyDjo28URppPCegVj1oLk-9u5oEPWzcrLspApQ9W4mavJEzHrufOti7RHWQPF9warw1vZI8vaplugiojzxjHuw8LSXrPH0v9bwwmoM9Dt6CPPIufTwHu_G8XK8qve6psrrvHtI60l8Rvdwt6byZj7M8VVMDPI3nxboqd7M84gFXu4-rI71DE688wEXxPIxfsLycKq07rWrJvAowJb1mG_28M-YRvYUWtTxq1Da95v7DvJHRKj0VJxC7QU_sPEdyxbyw8hY9xUIWPaYhPDtVtay7faepvBg3Tb1Ai2E8Kuz2vNOrebygnF09X13QvEN1xjy4dM67XE0cvE67Lr0yhCe9rbmiPZvuf7yT2YW8_E5DvR8TXr1QVt46iLGuPJqXTb3Z7DY9LunjPNkw2Lw3uj499ZCEPbLJl7vefAw9TWznvJ8nPr0OLZI8Ff7Yva0IoLsEmIg9x__juwaoe7vmmgY97vgdPUIAuTwOQCw79-_0O_K5ubxN96M8GkqNPY2YbL0bcJ08y3gtPK0IoDucAfY8uLAfPbDceb0c0iK9ysdgOQgKFbtkCwq-NqfIvGD7Aj3cuEk8z8QFO7CjKzwNyzA9aslsPdRv8ryQq4i8e7LEPLaKD7v7KI-9fJSzPO6WmLy27Dg8TNGTvOSyIz0arLa9zRNdPW7G_TxUUzm958JOPbrWCz3L2kS9tU50PZ3ut7yenwS9rQggvH_1-LxCTxK8OeAqPSvsUr3Ijja9Tn9LPYixrjuD3Qq9HzyVPGgbNbzyV7S8UQerOj5l9bymITw8H4BIu3g1Hb3KePU8F9VQvZvIp7yN50W8poNlvB5alL3M7cy8IUSLO0HaH70NLba8PI70vGq2QLwRjMy8YF0aPZmPIbzVvqe9MHF5PcjdobuWuM08rQggvaMRiL1Tokg97-JcvMgsjb0-8A09bNysPbrpJTxQQyC9e-NUvF8OZTz_Xve85CdDPEXB-DyPb9s8XdVePVRAjbx3SFu8lM4pvHTWhjwgk746a2dDvF2vBjw00Gs9akEhPE3kCT2PmJK8D2bgPIS0Qb2xRY-8-jdbPQuloLvyzFO9uRKKPHaEdD12Xhy8Je0Uve-TcT1lMRo9PlI3PafSCLhy1nK8qLzrPA0E_zzkJ0M8eUiTPf1hubxkqYS9IhvUPPeNyzz7O006wmvdOxX-2L0EDcy8zcRxvTp-gT0f4nE7u3HfPB3lBj2Pq6w8FHZDvZlANj0OQKw7z3WavIVlMj1B2qi7jP0GvMBF8btDOWM8w-B8PX4cpb2tkP08YqnwPHLW8jzPJgs9-mSEvbf_rrzN7QS845-tvP3_j72M_Ya7ZkQ0vczajryGKTS9RMSNvS-Hnz3aQ_I8IIAkPbV3vTx8Mgq9z4i0vP_pTr1UZmU8lDDTOwl_ND1zdKU8G3AdPUlJIr0aIfo77x7Su1Zmi7wmK7e90oVFvSMuSj0WwuO85WvJPKa_ErwarD88mFN0OkJPpLrB0Am8hviavMhl_7xIIxK98sxTPKZwJzz8YW89lGwAvFIaIbx2QLi8sY3qvMTzcj37O006BPoxvVrFjzwPU6K81tFBPGzcrLqmg-W87ZY8vObYj7wEvjy9cZ0ku48Nsjtmpl09Z71gvcOnCrzPxAW9e24RPWUeAL3WIAm9YHDYPCTflrrcuEk8Q2KaPS-tyrze3hG936JAvGhqoDy_gWa9v1uyOw6PFz2QglG86DfuPEX9yTzazhw9sFSuPCEI3ru4_wq9QrHNPDwZDTz5UTK9DS02vYLw2jy_qh09v1syPNP6Ur0HMwu9HVpcvVgU57wdNJY8uvy_PO6AezxBPK68vHG7vEHahLvnr7S9DAcCvdWCerz9Es67lB25vO-T8bsUFJq84oyTvV8O5TsCIzG7gcrKPH5rkLvQrsQ8KaCOvCXJ1bsmBQO9XP4nPesle71QMJi8qm0Uu8YZTb3nwjw9C2lzvOSyI7xXPcI8sHp0PZZDk7xvUQS9-cZRPZCC0TvoERY9mrVVPOvS-Twez_u72jA0vSo74rwOQCy7nrIMvYcA4juuzKq8C0MJPXRLbj2yjU88FMUuvSv_bD3_1pC9yfAXva_yOj2tCKC769JePRF5DjzIixi9gBl-Pe-8KD0G5Ci9Cc7DPKxEJz1grJc9La4KvaERWb1nppU7rd_oPDdFn7yatag8HSnDvIxfDD0VTcQ83_EHPeBmJ709LCe99qOMvApDP72cAfa8YCHbPGpBITpXsuG8qfiGvKNzBD2QXAI9o5luvXGKijzaQ_K8R5h5PVJpjLug_mI9ZVfyPC_WAT0AI3m9", } tabbed_examples = { "Pups": { "Pup1": "pup1.jpg", "Prompt": "Teacup Yorkies", "Pup2": "pup2.jpg", "Pup3": "pup3.jpg", "Pup4": "pup4.jpeg", "Pup5": "pup5.jpg", }, "Prototypes": { "Black & White": "#Black & White Embedding Prototype", "Fire": "#Fire Embedding Prototype", }, "Portraits": { "Snoop": "Snoop Dogg.jpg", "Snoop Prompt": "Snoop Dogg", "Ray": "Ray-Liotta-Goodfellas.jpg", "Ray Prompt": "Ray Liotta, Goodfellas", "Anya": "Anya Taylor-Joy 003.jpg", "Anya Prompt": "Anya Taylor-Joy, The Queen's Gambit", "Billie": "billie eilish 004.jpeg", "Billie Prompt": "Billie Eilish, blonde hair", "Lizzo": "Lizzo 001.jpeg", "Lizzo Prompt": "Lizzo,", "Donkey": "Donkey.jpg", "Donkey Prompt": "Donkey, from Shrek", }, "Transforms": { "ColorWheel001": "ColorWheel001.jpg", "ColorWheel001 BW": "ColorWheel001 BW.jpg", "ColorWheel002": "ColorWheel002.jpg", "ColorWheel002 BW": "ColorWheel002 BW.jpg", }, "CoCo": { "452650": "452650.jpeg", "Prompt 1": "a college dorm with a desk and bunk beds", "371739": "371739.jpeg", "Prompt 2": "a large banana is placed before a stuffed monkey.", "557922": "557922.jpeg", "Prompt 3": "a person sitting on a bench using a cell phone", "540554": "540554.jpeg", "Prompt 4": "two trains are coming down the tracks, a steam engine and a modern train.", }, # "NFT's": { # "SohoJoe": "SohoJoeEth.jpeg", # "SohoJoe Prompt": "SohoJoe.Eth", # "Mirai": "Mirai.jpg", # "Mirai Prompt": "Mirai from White Rabbit, @shibuyaxyz", # "OnChainMonkey": "OnChainMonkey-2278.jpg", # "OCM Prompt": "On Chain Monkey", # "Wassie": "Wassie 4498.jpeg", # "Wassie Prompt": "Wassie by Wassies", # }, } tile_size = 110 image_examples_tile_size = 50 with gr.Blocks() as demo: with gr.Row(): with gr.Column(scale=5): gr.Markdown( """ # Soho-Clip Embeddings Explorer A tool for exploring CLIP embedding space. **Example #1** - removing the Teacup from the image * Add the image Pups->Pup1 on Input tab 1 * Add the text prompt "Teacup." on Input tab 2 * Make the Input 2 embeddings negative by setting the power to -1 * Click the "Search Embedding Space" to see the results """) with gr.Column(scale=2, min_width=(tile_size+20)*3): with gr.Row(): with gr.Column(scale=1, min_width=tile_size): gr.Markdown("#### Pup in cup:") with gr.Column(scale=1, min_width=tile_size): gr.Markdown("#### - 'Teacup'") with gr.Column(scale=1, min_width=tile_size): gr.Markdown("#### = Pup") for example in examples: with gr.Row(): for example in example: with gr.Column(scale=1, min_width=tile_size): if len(example): local_path = os.path.join(image_folder, example) gr.Image( value = local_path, shape=(tile_size,tile_size), show_label=False, interactive=False) \ .style(height=tile_size, width=tile_size) with gr.Row(): for i in range(max_tabs): with gr.Tab(f"Input {i+1}"): with gr.Row(): with gr.Column(scale=1, min_width=240): input_images[i] = gr.Image(label="Image Prompt", show_label=True) with gr.Column(scale=3, min_width=600): embedding_plots[i] = gr.LinePlot(show_label=False).style(container=False) # input_image.change(on_image_load, inputs= [input_image, plot]) with gr.Row(): with gr.Column(scale=2, min_width=240): input_prompts[i] = gr.Textbox(label="Text Prompt", show_label=True, max_lines=4) with gr.Column(scale=3, min_width=600): with gr.Row(): # with gr.Slider(min=-5, max=5, value=1, label="Power", show_label=True): # embedding_powers[i] = gr.Slider.value embedding_powers[i] = gr.Slider(minimum=-3, maximum=3, value=1, label="Power", show_label=True, interactive=True) with gr.Row(): with gr.Accordion(f"Embeddings (base64)", open=False): embedding_base64s[i] = gr.Textbox(show_label=False, live=True) for idx, (tab_title, examples) in enumerate(tabbed_examples.items()): with gr.Tab(tab_title): with gr.Row(): for idx, (title, example) in enumerate(examples.items()): if example.endswith(".jpg") or example.endswith(".jpeg"): # add image example local_path = os.path.join(image_folder, example) with gr.Column(scale=1, min_width=image_examples_tile_size): gr.Examples( examples=[local_path], inputs=input_images[i], label=title, ) else: # add text example with gr.Column(scale=1, min_width=image_examples_tile_size*2): gr.Examples( examples=[example], inputs=input_prompts[i], label=title, ) with gr.Row(): average_embedding_plot = gr.LinePlot(show_label=True, label="Average Embeddings (base64)").style(container=False) with gr.Row(): with gr.Accordion(f"Avergage embeddings in base 64", open=False): average_embedding_base64 = gr.Textbox(show_label=False) with gr.Row(): with gr.Column(scale=1, min_width=200): n_samples = gr.Slider(1, 16, value=4, step=1, label="Number images") with gr.Column(scale=3, min_width=200): submit = gr.Button("Search embedding space") # with gr.Row(): # output = gr.Gallery(label="Closest images in Laion 5b using kNN", show_label=True)\ # .style(grid=[4,4], height="auto") output = gr.Gallery(label="Closest images in Laion 5b using kNN", show_label=True)\ .style(grid=[4,4], height="auto") submit.click(main, inputs= [average_embedding_base64, n_samples], outputs=output) embedding_base64s_state = gr.State(value=[None for i in range(max_tabs)]) embedding_power_state = gr.State(value=[1. for i in range(max_tabs)]) def on_image_load(input_image, idx_state, embedding_base64s_state, embedding_power_state): debug_print("on_image_load") embeddings_b64 = on_image_load_update_embeddings(input_image) new_plot = on_embeddings_changed_update_plot(embeddings_b64) average_embeddings_b64 = on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embeddings_b64, idx_state) new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64) return embeddings_b64, new_plot, average_embeddings_b64, new_average_plot def on_prompt_change(prompt, idx_state, embedding_base64s_state, embedding_power_state): debug_print("on_prompt_change") if is_prompt_embeddings(prompt): embeddings_b64 = prompt elif is_prompt_in_example_embeddings(prompt): embeddings_b64 = example_embeddings[prompt] else: embeddings_b64 = on_prompt_change_update_embeddings(prompt) new_plot = on_embeddings_changed_update_plot(embeddings_b64) average_embeddings_b64 = on_embeddings_changed_update_average_embeddings(embedding_base64s_state, embedding_power_state, embeddings_b64, idx_state) new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64) return embeddings_b64, new_plot, average_embeddings_b64, new_average_plot def on_power_change(power, idx_state, embedding_base64s_state, embedding_power_state): debug_print("on_power_change") average_embeddings_b64 = on_power_change_update_average_embeddings(embedding_base64s_state, embedding_power_state, power, idx_state) new_average_plot = on_embeddings_changed_update_plot(average_embeddings_b64) return average_embeddings_b64, new_average_plot for i in range(max_tabs): idx_state = gr.State(value=i) input_images[i].change(on_image_load, [input_images[i], idx_state, embedding_base64s_state, embedding_power_state], [embedding_base64s[i], embedding_plots[i], average_embedding_base64, average_embedding_plot]) input_prompts[i].change(on_prompt_change, [input_prompts[i], idx_state, embedding_base64s_state, embedding_power_state], [embedding_base64s[i], embedding_plots[i], average_embedding_base64, average_embedding_plot]) embedding_powers[i].change(on_power_change, [embedding_powers[i], idx_state, embedding_base64s_state, embedding_power_state], [average_embedding_base64, average_embedding_plot]) with gr.Row(): gr.Markdown( """ My interest is to use CLIP for image/video understanding (see [CLIP_visual-spatial-reasoning](https://github.com/Sohojoe/CLIP_visual-spatial-reasoning).) **Example #2** - adding black & white embedding prototype * Add the image Pups->Pup4 on Input tab 1 * Add Prototypes->Black&White on Input tab 2 * Set Input 2 embeddings power to 1.3 * Click the "Search Embedding Space" to see the results * Note: You may need to play with the power with different source images ### Initial Features - Combine up to 10 Images and/or text inputs to create an average embedding space. - Search the laion 5b images via a kNN search ### Known limitations - I'm getting formatting bugs when running on Huggingface (vs my Mac Book). This is impacting: - The galary - The Embeddings Tab ### Acknowledgements - I heavily build on [clip-retrieval](https://rom1504.github.io/clip-retrieval/) and use their API. Please [cite](https://github.com/rom1504/clip-retrieval#citation) the authors if you use this work. - [CLIP](https://openai.com/blog/clip/) - [Stable Diffusion](https://github.com/CompVis/stable-diffusion) """) # # Options # with gr.Row(): # gr.Markdown( # """ # ### Options # """) # with gr.Row(): # combined_embedding_mode = gr.Radio(choices=["Sum", "Average"], label="How to combine embedding", show_label=True, value="Sum", interactive=True) # API with gr.Row(): gr.Markdown( """ ### API UI elements to mock out the API """) with gr.Row(): _input = gr.Textbox(label="Input") with gr.Accordion(f"Embeddings (base64)", open=False): _output = gr.Textbox(value="", lines=2, label="Output") _btn = gr.Button(value="Submit") _btn.click(on_prompt_change_update_embeddings, inputs=_input, outputs=[_output], api_name="text_to_embeddings") # input_prompts[i] = gr.Textbox(label="Text Prompt", show_label=True, max_lines=4, api_name="text_to_embeddings") with gr.Row(): _input = gr.Image(label="Image Prompt", show_label=True) with gr.Accordion(f"Embeddings (base64)", open=False): _output = gr.Textbox(value="", lines=2, label="Output") _btn = gr.Button(value="Submit") _btn.click(on_image_load_update_embeddings, inputs=_input, outputs=[_output], api_name="image_to_embeddings") with gr.Row(): def _on_image_load(input_image): debug_print("_on_image_load") # resize if size is bigger than clip_image_size if input_image.shape[0] > clip_image_size or input_image.shape[1] > clip_image_size: input_image = preprocess_image(input_image, clip_image_size) payload = encode_numpy_array(input_image) return payload _input = gr.Image(label="Image Prompt", show_label=True) with gr.Accordion(f"Image (base64)", open=False): _input_as_texct = gr.Textbox(value="", lines=2, label="Output") _input.change(_on_image_load, inputs=_input, outputs=[_input_as_texct]) with gr.Accordion(f"Embeddings (base64)", open=False): _output = gr.Textbox(value="", lines=2, label="Output") _btn = gr.Button(value="Submit") _btn.click(on_image_as_payload_update_embeddings, inputs=_input_as_texct, outputs=[_output], api_name="image_as_payload_to_embeddings") # ![Alt Text](file/pup1.jpg) # # ![Alt Text](file/pup1.jpg){height=100 width=100} if __name__ == "__main__": demo.launch(debug=True)