import io import streamlit as st from model import * # # TODO: # - Reformat the model introduction # - Make the iterative text generation def gen_show_caption(sub_prompt=None, cap_prompt=""): with st.spinner("Generating Caption"): subreddit, caption = virtexModel.predict( image_dict, sub_prompt=sub_prompt, prompt=cap_prompt ) st.markdown( f""" r/{subreddit}: {cap_prompt} {caption} """, unsafe_allow_html=True, ) with st.spinner("Loading Model"): virtexModel, imageLoader, sample_images, valid_subs = create_objects() # ---------------------------------------------------------------------------- # Populate sidebar. # ---------------------------------------------------------------------------- select_idx = None st.sidebar.title("Select or upload an image") if st.sidebar.button("Random Sample Image"): select_idx = get_rand_idx(sample_images) sample_image = sample_images[0 if select_idx is None else select_idx] uploaded_image = None uploaded_file = st.sidebar.file_uploader("Choose a file") if uploaded_file is not None: uploaded_image = Image.open(io.BytesIO(uploaded_file.getvalue())) select_idx = None # Set this to help rewrite the cache st.sidebar.title("Select a Subreddit") sub = st.sidebar.selectbox( "Type below to condition on a subreddit. Select None for a predicted subreddit", valid_subs, ) st.sidebar.title("Write a Custom Prompt") cap_prompt = st.sidebar.text_input("Write the start of your caption below", value="") _ = st.sidebar.button("Regenerate Caption") st.sidebar.title("Advanced Options") num_captions = st.sidebar.select_slider( "Number of Captions to Predict", options=[1, 2, 3, 4, 5], value=1 ) nuc_size = st.sidebar.slider( "Nucleus Size:\nLarger values lead to more diverse captions", min_value=0.0, max_value=1.0, value=0.8, step=0.05, ) # ---------------------------------------------------------------------------- virtexModel.model.decoder.nucleus_size = nuc_size image_file = sample_image # LOAD AND CACHE THE IMAGE if uploaded_image is not None: image = uploaded_image elif select_idx is None and "image" in st.session_state: image = st.session_state["image"] else: image = Image.open(image_file) image = image.convert("RGB") st.session_state["image"] = image image_dict = imageLoader.transform(image) show_image = imageLoader.show_resize(image) st.title("Image Captioning with VirTex model trained on RedCaps") st.markdown(""" Caption your own images or try out some of our sample images. You can also generate captions as if they are from specific subreddits, as if they start with a particular prompt, or even both. Tweet your results with `#redcaps`! **Note:** This model was not trained on images of people, hence may not generate accurate captions describing humans. For more details, visit [redcaps.xyz](https://redcaps.xyz) check out our [NeurIPS 2021 paper](https://openreview.net/forum?id=VjJxBi1p9zh). """) _, center, _ = st.columns([1, 10, 1]) with center: st.image(show_image) if sub is None and imageLoader.text_transform(cap_prompt) != "": st.write("Without a specified subreddit we default to /r/pics") for i in range(num_captions): gen_show_caption(sub, imageLoader.text_transform(cap_prompt))