Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import os | |
| from PIL import Image | |
| import base64 | |
| import requests | |
| from langchain.embeddings import HuggingFaceEmbeddings | |
| #from langchain.llms import OpenAI | |
| from langchain.chains.qa_with_sources import load_qa_with_sources_chain | |
| from langchain.docstore.document import Document | |
| from langchain.embeddings.openai import OpenAIEmbeddings | |
| from langchain.vectorstores.faiss import FAISS | |
| import pickle | |
| HUGGINGFACEHUB_API_TOKEN = os.environ["HUGGINGFACEHUB_API_TOKEN"] | |
| model_name = "sentence-transformers/all-mpnet-base-v2" | |
| hf = HuggingFaceEmbeddings(model_name=model_name) | |
| #Loading FAISS search index from disk | |
| #This is a vector space of embeddings from one-tenth of PlaygrondAI image-prompts | |
| #PlaygrondAI open-sourced dataset is a collection of around 1.3 mil generated images and caption pairs | |
| with open("search_index0.pickle", "rb") as f: | |
| search_index = pickle.load(f) | |
| with open("search_index1.pickle", "rb") as f: | |
| search_index1 = pickle.load(f) | |
| with open("search_index2.pickle", "rb") as f: | |
| search_index2 = pickle.load(f) | |
| with open("search_index3.pickle", "rb") as f: | |
| search_index3 = pickle.load(f) | |
| #Defining methods for inference | |
| def encode(img): | |
| #Encode source image file to base64 string | |
| with open(img, "rb") as image_file: | |
| encoded_string = base64.b64encode(image_file.read()).decode('utf-8') | |
| #Returning image as encoded string | |
| return encoded_string | |
| def get_caption(image_in): | |
| #Sending requests to BLIP2 Gradio-space API | |
| BLIP2_GRADIO_API_URL = "https://nielsr-comparing-captioning-models.hf.space/run/predict" | |
| response = requests.post(BLIP2_GRADIO_API_URL, json={ | |
| "data": ["data:image/jpg;base64," + encode(image_in) ] | |
| }).json() | |
| data = response["data"][-1] | |
| return data | |
| def Image_similarity_search(image_in, search_query): | |
| if search_query == '': | |
| #Get image caption from Bip2 Gradio space | |
| img_caption = get_caption(image_in) | |
| else: | |
| img_caption = search_query | |
| print(f"Image caption from Blip2 Gradio Space or the search_query is - {img_caption}") | |
| #Searching the vector space | |
| search_result = search_index.similarity_search(img_caption)[0] | |
| search_result1 = search_index1.similarity_search(img_caption)[0] | |
| search_result2 = search_index2.similarity_search(img_caption)[0] | |
| search_result3 = search_index3.similarity_search(img_caption)[0] | |
| #Formatting the search results | |
| pai_prompt = list(search_result)[0][1] | |
| pai_prompt1 = list(search_result1)[0][1] | |
| pai_prompt2 = list(search_result2)[0][1] | |
| pai_prompt3 = list(search_result3)[0][1] | |
| pai_img_link = list(search_result)[-2][-1]['source'] | |
| pai_img_link1 = list(search_result1)[-2][-1]['source'] | |
| pai_img_link2 = list(search_result2)[-2][-1]['source'] | |
| pai_img_link3 = list(search_result3)[-2][-1]['source'] | |
| html_tag = f"""<div style="display: flex; flex-direction: row; overflow-x: auto;"> | |
| <img src='{pai_img_link}' alt='{img_caption}' style='display: block; margin: auto;'> | |
| <img src='{pai_img_link1}' alt='{img_caption}' style='display: block; margin: auto;'> | |
| <img src='{pai_img_link2}' alt='{img_caption}' style='display: block; margin: auto;'> | |
| <img src='{pai_img_link3}' alt='{img_caption}' style='display: block; margin: auto;'> | |
| </div>""" #class='gallery' > | |
| return html_tag #pai_prompt | |
| #Defining Gradio Blocks | |
| with gr.Blocks(css = """#label_mid {padding-top: 2px; padding-bottom: 2px;} | |
| #label_results {padding-top: 5px; padding-bottom: 1px;} | |
| #col-container {max-width: 580px; margin-left: auto; margin-right: auto;} | |
| #accordion {max-width: 580px; margin-left: auto; margin-right: auto;} | |
| #img_search img {margin: 10px; max-width: 300px; max-height: 300px;} | |
| """) as demo: | |
| gr.HTML("""<div style="text-align: center; max-width: 700px; margin: 0 auto;"> | |
| <div | |
| style=" | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 0.8rem; | |
| font-size: 1.75rem; | |
| " | |
| > | |
| <h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;"> | |
| Using Gradio Demos as API</h1><br></div> | |
| <div><h4 style="font-weight: 500; margin-bottom: 7px; margin-top: 5px;"> | |
| Get BLIP2 captions from <a href="https://langchain.readthedocs.io/en/latest/" target="_blank">Niels space</a> via API call,<br> | |
| Use LangChain to create vector space with PlaygroundAI prompts</h4><br> | |
| </div>""") | |
| with gr.Accordion(label="Details about the working:", open=False, elem_id='accordion'): | |
| gr.HTML(""" | |
| <p style="margin-bottom: 10px; font-size: 90%"><br> | |
| ▶️Do you see the "view api" link located in the footer of this application? | |
| By clicking on this link, a page will open which provides documentation on the REST API that developers can use to query the Interface function / Block events.<br> | |
| ▶️In this demo, the first step involves making an API call to the BLIP2 Gradio demo to retrieve image captions. | |
| Next, Langchain is used to create an embedding and vector space for the image prompts and their respective "source" from the PlaygroundAI dataset. | |
| Finally, a similarity search is performed over the vector space and the top result is returned. | |
| </p></div>""") | |
| #with gr.Column(scale=3): | |
| # pass | |
| with gr.Column(elem_id = "col-container"): | |
| label_top = gr.HTML(value= "<center>🖼️ Please upload an Image here👇 that will be used as your search query</center>", elem_id="label_top") | |
| image_in = gr.Image(label="Upoload an Image for search", type='filepath', elem_id="image_in") | |
| label_mid = gr.HTML(value= "<p style='text-align: center; color: red;'>Or</center></p>", elem_id='label_mid') | |
| label_bottom = gr.HTML(value= "<center>🔍Type in your serch query and press Enter 👇</center>", elem_id="label_bottom") | |
| search_query = gr.Textbox(placeholder="Example: A small cat sitting", label="", elem_id="search_query", value='') | |
| label_results = gr.HTML(value= "<p style='text-align: center; color: blue; font-weight: bold;'>👇These Search results are from PlaygroundAI 'Liked_Images' dataset available on <a href='https://github.com/playgroundai/liked_images' _target='blank'>github</a></center></p>", elem_id="label_results") | |
| img_search = gr.HTML(label = 'Image search results from PlaygroundAI dataset', elem_id="img_search") | |
| #pai_prompt = gr.Textbox(label="Image prompt from PlaygroundAI dataset", elem_id="pai_prompt") | |
| #b1 = gr.Button("Retry").style(full_width=False) | |
| gr.HTML('''<center><a href="https://huggingface.co/spaces/ysharma/Blip_PlaygroundAI?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a></center> | |
| </p></div>''') | |
| image_in.change(Image_similarity_search, [image_in, search_query], [img_search], api_name="PlaygroundAI_image_search" ) | |
| search_query.submit(Image_similarity_search, [image_in, search_query], [img_search], api_name='PlaygroundAI_text_search' ) | |
| #b1.click(Image_similarity_search, [image_in, search_query], [pai_prompt, img_search] ) | |
| demo.launch(debug=True) |