File size: 2,912 Bytes
0fc1ca7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
"""
# Copyright (c) 2022, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""
import numpy as np
import streamlit as st
import torch
from lavis.models.blip_models.blip_image_text_matching import compute_gradcam
from lavis.processors import load_processor
from PIL import Image
from app import device, load_demo_image
from app.utils import getAttMap, init_bert_tokenizer, load_blip_itm_model
def app():
model_type = st.sidebar.selectbox("Model:", ["BLIP_base", "BLIP_large"])
if model_type.startswith("BLIP"):
blip_type = model_type.split("_")[1]
model = load_blip_itm_model(device, model_type=blip_type)
vis_processor = load_processor("blip_image_eval").build(image_size=384)
st.markdown(
"<h1 style='text-align: center;'>Image Text Matching</h1>",
unsafe_allow_html=True,
)
values = list(range(1, 12))
default_layer_num = values.index(7)
layer_num = (
st.sidebar.selectbox("Layer number", values, index=default_layer_num) - 1
)
instructions = """Try the provided image or upload your own:"""
file = st.file_uploader(instructions)
col1, col2 = st.columns(2)
col1.header("Image")
col2.header("GradCam")
if file:
raw_img = Image.open(file).convert("RGB")
else:
raw_img = load_demo_image()
w, h = raw_img.size
scaling_factor = 720 / w
resized_image = raw_img.resize((int(w * scaling_factor), int(h * scaling_factor)))
col1.image(resized_image, use_column_width=True)
col3, col4 = st.columns(2)
col3.header("Text")
user_question = col3.text_input(
"Input your sentence!", "a woman sitting on the beach with a dog"
)
submit_button = col3.button("Submit")
col4.header("Matching score")
if submit_button:
tokenizer = init_bert_tokenizer()
img = vis_processor(raw_img).unsqueeze(0).to(device)
text_processor = load_processor("blip_caption").build()
qry = text_processor(user_question)
norm_img = np.float32(resized_image) / 255
qry_tok = tokenizer(qry, return_tensors="pt").to(device)
gradcam, output = compute_gradcam(model, img, qry, qry_tok, block_num=layer_num)
avg_gradcam = getAttMap(norm_img, gradcam[0][1], blur=True)
col2.image(avg_gradcam, use_column_width=True, clamp=True)
# output = model(img, question)
itm_score = torch.nn.functional.softmax(output, dim=1)
new_title = (
'<p style="text-align: left; font-size: 25px;">\n{:.3f}%</p>'.format(
itm_score[0][1].item() * 100
)
)
col4.markdown(new_title, unsafe_allow_html=True)
|