Merge branch 'main' of https://huggingface.co/spaces/flax-community/multilingual-image-captioning into main

.gitignore

@@ -1,3 +1,2 @@
-*mic_env/*
-**__pycache__**
-*.pyc
+mic_env/*
+*.pyc

app.py

@@ -44,9 +44,9 @@ code_to_name = {
 }
 
 @st.cache(persist=True)
-def generate_sequence(pixel_values, lang_code, num_beams):
+def generate_sequence(pixel_values, lang_code, num_beams, temperature, top_p):
     lang_code = language_mapping[lang_code]
-    output_ids = model.generate(input_ids=pixel_values, forced_bos_token_id=tokenizer.lang_code_to_id[lang_code], max_length=64, num_beams=num_beams)
+    output_ids = model.generate(input_ids=pixel_values, forced_bos_token_id=tokenizer.lang_code_to_id[lang_code], max_length=64, num_beams=num_beams, temperature=temperature, top_p = top_p)
     print(output_ids)
     output_sequence = tokenizer.batch_decode(output_ids[0], skip_special_tokens=True, max_length=64)
     return output_sequence
@@ -56,13 +56,14 @@ def read_markdown(path, parent="./sections/"):
         return f.read()
 
 
-checkpoints = ["./ckpt/ckpt-22499"]  # TODO: Maybe add more checkpoints?
+checkpoints = ["./ckpt/ckpt-17499"]  # TODO: Maybe add more checkpoints?
 dummy_data = pd.read_csv("reference.tsv", sep="\t")
 
 st.set_page_config(
     page_title="Multilingual Image Captioning",
     layout="wide",
     initial_sidebar_state="collapsed",
+    page_icon="./misc/mic-logo.png",
 )
 
 st.title("Multilingual Image Captioning")
@@ -70,12 +71,33 @@ st.write(
     "[Bhavitvya Malik](https://huggingface.co/bhavitvyamalik), [Gunjan Chhablani](https://huggingface.co/gchhablani)"
 )
 
-st.sidebar.title("Settings")
+st.sidebar.title("Generation Parameters")
 num_beams = st.sidebar.number_input(label="Number of Beams", min_value=2, max_value=10, value=4, step=1, help="Number of beams to be used in beam search.")
+temperature = st.sidebar.select_slider(label="Temperature", options = list(np.arange(0.0,1.1, step=0.1)), value=1.0, help ="The value used to module the next token probabilities.", format_func=lambda x: f"{x:.2f}")
+top_p = st.sidebar.select_slider(label = "Top-P", options = list(np.arange(0.0,1.1, step=0.1)),value=1.0, help="Nucleus Sampling : If set to float < 1, only the most probable tokens with probabilities that add up to :obj:`top_p` or higher are kept for generation.", format_func=lambda x: f"{x:.2f}")
+
+image_col, intro_col = st.beta_columns([3, 8])
+image_col.image("./misc/mic-logo.png", use_column_width="always")
+intro_col.write(read_markdown("intro.md"))
 
 with st.beta_expander("Usage"):
     st.markdown(read_markdown("usage.md"))
 
+with st.beta_expander("Article"):
+    st.write(read_markdown("abstract.md"))
+    st.write(read_markdown("caveats.md"))
+    # st.write("# Methodology")
+    # st.image(
+    #     "./misc/Multilingual-IC.png", caption="Seq2Seq model for Image-text Captioning."
+    # )
+    st.markdown(read_markdown("pretraining.md"))
+    st.write(read_markdown("challenges.md"))
+    st.write(read_markdown("social_impact.md"))
+    st.write(read_markdown("references.md"))
+    # st.write(read_markdown("checkpoints.md"))
+    st.write(read_markdown("acknowledgements.md"))
+
+
 first_index = 20
 # Init Session State
 if state.image_file is None:
@@ -87,9 +109,9 @@ if state.image_file is None:
     image = plt.imread(image_path)
     state.image = image
 
-col1, col2 = st.beta_columns([6, 4])
+# col1, col2 = st.beta_columns([6, 4])
 
-if col2.button("Get a random example"):
+if st.button("Get a random example", help="Get a random example from one of the seeded examples."):
     sample = dummy_data.sample(1).reset_index()
     state.image_file = sample.loc[0, "image_file"]
     state.caption = sample.loc[0, "caption"].strip("- ")
@@ -99,40 +121,42 @@ if col2.button("Get a random example"):
     image = plt.imread(image_path)
     state.image = image
 
-col2.write("OR")
+# col2.write("OR")
 
-uploaded_file = col2.file_uploader("Upload your image", type=["png", "jpg", "jpeg"])
-if uploaded_file is not None:
-    state.image_file = os.path.join("images", uploaded_file.name)
-    state.image = np.array(Image.open(uploaded_file))
+# uploaded_file = col2.file_uploader("Upload your image", type=["png", "jpg", "jpeg"])
+# if uploaded_file is not None:
+#     state.image_file = os.path.join("images", uploaded_file.name)
+#     state.image = np.array(Image.open(uploaded_file))
 
 transformed_image = get_transformed_image(state.image)
 
+new_col1, new_col2 = st.beta_columns([5,5])
 # Display Image
-col1.image(state.image, use_column_width="auto")
+new_col1.image(state.image, use_column_width="always")
+
 
 # Display Reference Caption
-col2.write("**Reference Caption**: " + state.caption)
-col2.markdown(
+new_col2.write("**Reference Caption**: " + state.caption)
+new_col2.markdown(
     f"""**English Translation**: {state.caption if state.lang_id == "en" else translate(state.caption, 'en')}"""
 )
 
 # Select Language
 options = list(code_to_name.keys())
-lang_id = col2.selectbox(
+lang_id = new_col2.selectbox(
     "Language",
     index=options.index(state.lang_id),
     options=options,
     format_func=lambda x: code_to_name[x],
+    help="The language in which caption is to be generated."
 )
-# Display Top-5 Predictions
+
 with st.spinner("Loading model..."):
     model = load_model(checkpoints[0])
-
 sequence = ['']
-if col2.button("Generate Caption"):
+if new_col2.button("Generate Caption", help="Generate a caption in the specified language."):
     with st.spinner("Generating Sequence..."):
-        sequence = generate_sequence(transformed_image, lang_id, num_beams)
+        sequence = generate_sequence(transformed_image, lang_id, num_beams, temperature, top_p)
 # print(sequence)
 
 if sequence!=['']:
@@ -143,15 +167,3 @@ if sequence!=['']:
     st.write(
         "**English Translation**: "+ sequence[0] if lang_id=="en" else translate(sequence[0])
     )
-st.write(read_markdown("abstract.md"))
-st.write(read_markdown("caveats.md"))
-# st.write("# Methodology")
-# st.image(
-#     "./misc/Multilingual-IC.png", caption="Seq2Seq model for Image-text Captioning."
-# )
-st.markdown(read_markdown("pretraining.md"))
-st.write(read_markdown("challenges.md"))
-st.write(read_markdown("social_impact.md"))
-st.write(read_markdown("references.md"))
-# st.write(read_markdown("checkpoints.md"))
-st.write(read_markdown("acknowledgements.md"))