File size: 4,279 Bytes
bc7d231 c7c92f9 dfda773 58e3cb5 63fc765 5554139 e9d7d81 fdc69a0 8cf7678 bc7d231 0fa8d68 8cf7678 1d94d91 1a7a04d 8cf7678 f35e4aa 2468667 0fa8d68 1d94d91 f35e4aa 2468667 8cf7678 f35e4aa 8cf7678 9e13523 8cf7678 f35e4aa 8cf7678 9d4c7bc 40e0ea9 9d4c7bc 40e0ea9 9d4c7bc 0fa8d68 9d4c7bc 0fa8d68 9d4c7bc 0fa8d68 9d4c7bc 0fa8d68 9d4c7bc 8cf7678 45969bf 9d4c7bc 53fe45c 9d4c7bc c6252cf f35e4aa |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 |
import streamlit as st
import torch
import bitsandbytes
import accelerate
import scipy
from PIL import Image
import torch.nn as nn
from my_model.object_detection import detect_and_draw_objects
from my_model.captioner.image_captioning import get_caption
from my_model.utilities import free_gpu_resources
from my_model.KBVQA import KBVQA, prepare_kbvqa_model
def answer_question(image, question, model):
answer = model.generate_answer(question, image)
return answer
def get_caption(image):
return "Generated caption for the image"
def free_gpu_resources():
pass
# Sample images (assuming these are paths to your sample images)
sample_images = ["Files/sample1.jpg", "Files/sample2.jpg", "Files/sample3.jpg",
"Files/sample4.jpg", "Files/sample5.jpg", "Files/sample6.jpg",
"Files/sample7.jpg"]
def run_inference():
st.title("Run Inference")
# Button to load KBVQA models
if st.button('Load KBVQA Models'):
# Call the function to load models and show progress
kbvqa = prepare_kbvqa_model(your_detection_model) # Replace with your actual detection model
if kbvqa:
st.write("Model is ready for inference.")
image_qa_app(kbvqa)
def image_qa_app(kbvqa):
# Initialize session state for storing the current image and its Q&A history
if 'current_image' not in st.session_state:
st.session_state['current_image'] = None
if 'qa_history' not in st.session_state:
st.session_state['qa_history'] = []
# Display sample images as clickable thumbnails
st.write("Choose from sample images:")
cols = st.columns(len(sample_images))
for idx, sample_image_path in enumerate(sample_images):
with cols[idx]:
image = Image.open(sample_image_path)
if st.image(image, use_column_width=True):
st.session_state['current_image'] = image
st.session_state['qa_history'] = []
# Image uploader
uploaded_image = st.file_uploader("Or upload an Image", type=["png", "jpg", "jpeg"])
if uploaded_image is not None:
st.session_state['current_image'] = Image.open(uploaded_image)
st.session_state['qa_history'] = []
# Display the current image
if st.session_state['current_image'] is not None:
st.image(st.session_state['current_image'], caption='Uploaded Image.', use_column_width=True)
# Question input
question = st.text_input("Ask a question about this image:")
# Get Answer button
if st.button('Get Answer'):
# Process the question
answer = answer_question(st.session_state['current_image'], question, model=kbvqa)
free_gpu_resources()
st.session_state['qa_history'].append((question, answer))
# Display all Q&A
for q, a in st.session_state['qa_history']:
st.text(f"Q: {q}\nA: {a}\n")
# Main function
def main():
st.sidebar.title("Navigation")
selection = st.sidebar.radio("Go to", ["Home", "Dataset Analysis", "Evaluation Results", "Run Inference", "Dissertation Report", "Object Detection"])
if selection == "Home":
st.title("MultiModal Learning for Knowledg-Based Visual Question Answering")
st.write("Home page content goes here...")
elif selection == "Dissertation Report":
st.title("Dissertation Report")
st.write("Click the link below to view the PDF.")
# Example to display a link to a PDF
st.download_button(
label="Download PDF",
data=open("Files/Dissertation Report.pdf", "rb"),
file_name="example.pdf",
mime="application/octet-stream"
)
elif selection == "Evaluation Results":
st.title("Evaluation Results")
st.write("This is a Place Holder until the contents are uploaded.")
elif selection == "Dataset Analysis":
st.title("OK-VQA Dataset Analysis")
st.write("This is a Place Holder until the contents are uploaded.")
elif selection == "Run Inference":
run_inference()
elif selection == "Object Detection":
run_object_detection()
if __name__ == "__main__":
main()
|