app.py

import gradio as gr
import numpy as np
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer
import requests
from PIL import Image
from transformers import BlipProcessor, BlipForConditionalGeneration
sentence_model = SentenceTransformer("all-MiniLM-L6-v2")

processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
image_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")

def generate_input(input_type, image=None, text=None, response_amount=3):
    # initalize input variable 
    combined_input = ""

    # handle image input if chosen
    if input_type == "Image" and image:
        inputs = processor(images=image, return_tensors="pt") #process image with BlipProcessor
        out = image_model.generate(**inputs)  #generate caption with BlipModel
        image_caption = processor.decode(out[0], skip_special_tokens=True)  #decode output w/ processor
        combined_input += image_caption  # add the image caption to input

    # handle text input if chosen
    elif input_type == "Text" and text:
        combined_input += text  # add the text to input

    # handle both text and image input if chosen
    elif input_type == "Both" and image and text:
        inputs = processor(images=image, return_tensors="pt")
        out = image_model.generate(**inputs)
        image_caption = processor.decode(out[0], skip_special_tokens=True)  #repeat image processing + caption generation and decoding
        combined_input += image_caption + " and " + text  # combine image caption and text
    
    # if no input, fallback
    if not combined_input:
        combined_input = "No input provided."
    if response_amount is None:
        response_amount=3

    return vector_search(combined_input,response_amount) #search through embedded document w/ input

# load embeddings and metadata
embeddings = np.load("dog_data_embeddings.npy")  #created using sentence_transformers on kaggle
metadata = pd.read_csv("dog_metadata.csv") #created using sentence_transformers on kaggle

# vector search function
def vector_search(query,top_n=3):
    query_embedding = sentence_model.encode(query)  #encode input w/ Sentence Transformers
    similarities = cosine_similarity([query_embedding], embeddings)[0]  #similarity function
    if top_n is None:
        top_n=3
    top_indices = similarities.argsort()[-top_n:][::-1]  #return top n indices based on chosen output amount
    results = metadata.iloc[top_indices]  #get metadata
    result_text=""
    for index,row in results.iterrows():   #loop through results to get Title, Description, and Genre for top n outputs
        if index!=top_n-1:
            result_text+=f"Breed: {row['breed']}  Description: {row['description']}  Temperament: {row['temperament']} Energy Level: {row['energy_level_category']} Trainability: {row['trainability_category']} Demeanor:  {row['demeanor_category']} \n\n"
        else:
         result_text+=f"Breed: {row['breed']}  Description: {row['description']}  Temperament: {row['temperament']} Energy Level: {row['energy_level_category']} Trainability: {row['trainability_category']} Demeanor:  {row['demeanor_category']}"
    return result_text


def set_response_amount(response_amount):  #set response amount 
    if response_amount is None:
        return 3
    return response_amount

 # based on the selected input type, make the appropriate input visible
def update_inputs(input_type):
    if input_type == "Image":
        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
    elif input_type == "Text":
        return gr.update(visible=False), gr.update(visible=True), gr.update(visible=True)
    elif input_type == "Both":
        return gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
with gr.Blocks() as demo:
    gr.Markdown("# Dog Breed Recommendation System")
    gr.Markdown("Enter a query to receive dog breed recommendations based on description, temperament, trainability, and demeanor.")
   
    input_type = gr.Radio(["Image", "Text", "Both"], label="Select Input Type", type="value")
    response_type=gr.Dropdown(choices=[3,5,10,25], type="value", label="Select Response Amount", visible=False)
    image_input = gr.Image(label="Upload Image", type="pil", visible=False)  # Hidden initially
    text_input = gr.Textbox(label="Enter Text Query", placeholder="Enter a description or query here", visible=False)  # hidden initially
  
    input_type.change(fn=update_inputs, inputs=input_type, outputs=[image_input, text_input, response_type])
   # state variable to store the selected response amount
    selected_response_amount = gr.State()

    # capture response amount immediately when dropdown changes
    response_type.change(fn=set_response_amount, inputs=response_type, outputs=selected_response_amount)
    
    submit_button = gr.Button("Submit")
    output = gr.Textbox(label="Recommendations")
    if selected_response_amount is None:
        selected_response_amount=3

    submit_button.click(fn=generate_input, inputs=[input_type,image_input, text_input,selected_response_amount], outputs=output)
demo.launch()