multiapp

app.py

@@ -1,187 +1,200 @@
-from io import BytesIO
+from apps import article, mic
 import streamlit as st
-import pandas as pd
-import json
-import os
-import numpy as np
-from streamlit import caching
-from PIL import Image
-from model.flax_clip_vision_mbart.modeling_clip_vision_mbart import (
-    FlaxCLIPVisionMBartForConditionalGeneration,
-)
-from transformers import MBart50TokenizerFast
-from utils import (
-    get_transformed_image,
-)
-import matplotlib.pyplot as plt
-from mtranslate import translate
-
-
 from session import _get_state
-
-state = _get_state()
-
-
-@st.cache
-def load_model(ckpt):
-    return FlaxCLIPVisionMBartForConditionalGeneration.from_pretrained(ckpt)
-
-
-tokenizer = MBart50TokenizerFast.from_pretrained("facebook/mbart-large-50")
-
-language_mapping = {
-    "en": "en_XX",
-    "de": "de_DE",
-    "fr": "fr_XX",
-    "es": "es_XX"
-}
-
-code_to_name = {
-    "en": "English",
-    "fr": "French",
-    "de": "German",
-    "es": "Spanish",
-}
-
-@st.cache
-def generate_sequence(pixel_values, lang_code, num_beams, temperature, top_p, do_sample, top_k, max_length):
-    lang_code = language_mapping[lang_code]
-    output_ids = state.model.generate(input_ids=pixel_values, forced_bos_token_id=tokenizer.lang_code_to_id[lang_code], max_length=max_length, num_beams=num_beams, temperature=temperature, top_p = top_p, top_k=top_k, do_sample=do_sample)
-    print(output_ids)
-    output_sequence = tokenizer.batch_decode(output_ids[0], skip_special_tokens=True, max_length=max_length)
-    return output_sequence
-
-def read_markdown(path, parent="./sections/"):
-    with open(os.path.join(parent, path)) as f:
-        return f.read()
-
-
-checkpoints = ["./ckpt/ckpt-51999"]  # TODO: Maybe add more checkpoints?
-dummy_data = pd.read_csv("reference.tsv", sep="\t")
-
-st.set_page_config(
-    page_title="Multilingual Image Captioning",
-    layout="wide",
-    initial_sidebar_state="collapsed",
-    page_icon="./misc/mic-logo.png",
-)
-
-st.title("Multilingual Image Captioning")
-st.write(
-    "[Bhavitvya Malik](https://huggingface.co/bhavitvyamalik), [Gunjan Chhablani](https://huggingface.co/gchhablani)"
-)
-
-st.sidebar.title("Generation Parameters")
-# max_length = st.sidebar.number_input("Max Length", min_value=16, max_value=128, value=64, step=1, help="The maximum length of sequence to be generated.")
-max_length = 64
-do_sample = st.sidebar.checkbox("Sample", value=False, help="Sample from the model instead of using beam search.")
-top_k = st.sidebar.number_input("Top K", min_value=10, max_value=200, value=50, step=1, help="The number of highest probability vocabulary tokens to keep for top-k-filtering.")
-num_beams = st.sidebar.number_input(label="Number of Beams", min_value=2, max_value=10, value=4, step=1, help="Number of beams to be used in beam search.")
-temperature = st.sidebar.select_slider(label="Temperature", options = list(np.arange(0.0,1.1, step=0.1)), value=1.0, help ="The value used to module the next token probabilities.", format_func=lambda x: f"{x:.2f}")
-top_p = st.sidebar.select_slider(label = "Top-P", options = list(np.arange(0.0,1.1, step=0.1)),value=1.0, help="Nucleus Sampling : If set to float < 1, only the most probable tokens with probabilities that add up to :obj:`top_p` or higher are kept for generation.", format_func=lambda x: f"{x:.2f}")
-if st.sidebar.button("Clear All Cache"):
-    caching.clear_cache()
-image_col, intro_col = st.beta_columns([3, 8])
-image_col.image("./misc/mic-logo.png", use_column_width="always")
-intro_col.write(read_markdown("intro.md"))
-
-with st.beta_expander("Usage"):
-    st.markdown(read_markdown("usage.md"))
-
-with st.beta_expander("Article"):
-    st.write(read_markdown("abstract.md"))
-    st.write(read_markdown("caveats.md"))
-    st.write("## Methodology")
-    st.image(
-        "./misc/Multilingual-IC.png"
-    )
-    st.markdown(read_markdown("pretraining.md"))
-    st.write(read_markdown("challenges.md"))
-    st.write(read_markdown("social_impact.md"))
-    st.write(read_markdown("bias.md"))
-
-    col1, col2, col3, col4 = st.beta_columns([0.5,2.5,2.5,0.5])
-    with col2:
-        st.image("./misc/examples/female_dev_1.jpg", width=350, caption = 'German Caption: <PERSON> arbeitet an einem Computer.', use_column_width='always')
-    with col3:
-        st.image("./misc/examples/female_doctor.jpg", width=350, caption = 'English Caption: A portrait of <PERSON>, a doctor who specializes in health care.', use_column_width='always')
-
-    col1, col2, col3, col4 = st.beta_columns([0.5,2.5,2.5,0.5])
-    with col2:
-        st.image("./misc/examples/female_doctor_1.jpg", width=350, caption = 'Spanish Caption: El Dr. <PERSON> es un estudiante de posgrado.', use_column_width='always')
-    with col3:
-        st.image("./misc/examples/women_cricket.jpg", width=350, caption = 'English Caption: <PERSON> of India bats against <PERSON> of Australia during the first Twenty20 match between India and Australia at Indian Bowl Stadium in New Delhi on Friday. - PTI', use_column_width='always')
-
-    col1, col2, col3, col4 = st.beta_columns([0.5,2.5,2.5,0.5])
-    with col2:
-        st.image("./misc/examples/female_dev_2.jpg", width=350, caption = "French Caption: Un écran d'ordinateur avec un écran d'ordinateur ouvert.", use_column_width='always')
-    with col3:
-        st.image("./misc/examples/female_biker_resized.jpg", width=350, caption = 'German Caption: <PERSON> auf dem Motorrad von <PERSON>.', use_column_width='always')
-
-    st.write(read_markdown("future_scope.md"))
-    st.write(read_markdown("references.md"))
-    # st.write(read_markdown("checkpoints.md"))
-    st.write(read_markdown("acknowledgements.md"))
-
-if state.model is None:
-    with st.spinner("Loading model..."):
-        state.model = load_model(checkpoints[0])
-
-first_index = 25
-# Init Session State
-if state.image_file is None:
-    state.image_file = dummy_data.loc[first_index, "image_file"]
-    state.caption = dummy_data.loc[first_index, "caption"].strip("- ")
-    state.lang_id = dummy_data.loc[first_index, "lang_id"]
-
-    image_path = os.path.join("images", state.image_file)
-    image = plt.imread(image_path)
-    state.image = image
-
-if st.button("Get a random example", help="Get a random example from one of the seeded examples."):
-    sample = dummy_data.sample(1).reset_index()
-    state.image_file = sample.loc[0, "image_file"]
-    state.caption = sample.loc[0, "caption"].strip("- ")
-    state.lang_id = sample.loc[0, "lang_id"]
-
-    image_path = os.path.join("images", state.image_file)
-    image = plt.imread(image_path)
-    state.image = image
-
-transformed_image = get_transformed_image(state.image)
-
-new_col1, new_col2 = st.beta_columns([5,5])
-
-# Display Image
-new_col1.image(state.image, use_column_width="always")
-# Display Reference Caption
-with new_col1.beta_expander("Reference Caption"):
-    st.write("**Reference Caption**: " + state.caption)
-    st.markdown(
-        f"""**English Translation**: {state.caption if state.lang_id == "en" else translate(state.caption, 'en')}"""
+from multiapp import MultiApp
+
+# from io import BytesIO
+# from apps.utils import read_markdown
+# from apps import article
+# import streamlit as st
+# import pandas as pd
+# import os
+# import numpy as np
+# from streamlit import caching
+# from PIL import Image
+# from model.flax_clip_vision_mbart.modeling_clip_vision_mbart import (
+#     FlaxCLIPVisionMBartForConditionalGeneration,
+# )
+# import matplotlib.pyplot as plt
+# from mtranslate import translate
+
+
+# from session import _get_state
+
+# state = _get_state()
+
+
+# @st.cache
+# def load_model(ckpt):
+#     return FlaxCLIPVisionMBartForConditionalGeneration.from_pretrained(ckpt)
+
+
+# tokenizer = MBart50TokenizerFast.from_pretrained("facebook/mbart-large-50")
+
+# language_mapping = {
+#     "en": "en_XX",
+#     "de": "de_DE",
+#     "fr": "fr_XX",
+#     "es": "es_XX"
+# }
+
+# code_to_name = {
+#     "en": "English",
+#     "fr": "French",
+#     "de": "German",
+#     "es": "Spanish",
+# }
+
+# @st.cache
+# def generate_sequence(pixel_values, lang_code, num_beams, temperature, top_p, do_sample, top_k, max_length):
+#     lang_code = language_mapping[lang_code]
+#     output_ids = state.model.generate(input_ids=pixel_values, forced_bos_token_id=tokenizer.lang_code_to_id[lang_code], max_length=max_length, num_beams=num_beams, temperature=temperature, top_p = top_p, top_k=top_k, do_sample=do_sample)
+#     print(output_ids)
+#     output_sequence = tokenizer.batch_decode(output_ids[0], skip_special_tokens=True, max_length=max_length)
+#     return output_sequence
+
+
+# checkpoints = ["./ckpt/ckpt-51999"]  # TODO: Maybe add more checkpoints?
+# dummy_data = pd.read_csv("reference.tsv", sep="\t")
+
+
+# st.sidebar.title("Generation Parameters")
+# # max_length = st.sidebar.number_input("Max Length", min_value=16, max_value=128, value=64, step=1, help="The maximum length of sequence to be generated.")
+# max_length = 64
+# do_sample = st.sidebar.checkbox("Sample", value=False, help="Sample from the model instead of using beam search.")
+# top_k = st.sidebar.number_input("Top K", min_value=10, max_value=200, value=50, step=1, help="The number of highest probability vocabulary tokens to keep for top-k-filtering.")
+# num_beams = st.sidebar.number_input(label="Number of Beams", min_value=2, max_value=10, value=4, step=1, help="Number of beams to be used in beam search.")
+# temperature = st.sidebar.select_slider(label="Temperature", options = list(np.arange(0.0,1.1, step=0.1)), value=1.0, help ="The value used to module the next token probabilities.", format_func=lambda x: f"{x:.2f}")
+# top_p = st.sidebar.select_slider(label = "Top-P", options = list(np.arange(0.0,1.1, step=0.1)),value=1.0, help="Nucleus Sampling : If set to float < 1, only the most probable tokens with probabilities that add up to :obj:`top_p` or higher are kept for generation.", format_func=lambda x: f"{x:.2f}")
+# if st.sidebar.button("Clear All Cache"):
+#     caching.clear_cache()
+# image_col, intro_col = st.beta_columns([3, 8])
+# image_col.image("./misc/mic-logo.png", use_column_width="always")
+# intro_col.write(read_markdown("intro.md"))
+
+# with st.beta_expander("Usage"):
+#     st.markdown(read_markdown("usage.md"))
+
+# with st.beta_expander("Article"):
+#     st.write(read_markdown("abstract.md"))
+#     st.write("## Methodology")
+#     st.image(
+#         "./misc/Multilingual-IC.png"
+#     )
+#     st.markdown(read_markdown("pretraining.md"))
+#     st.write(read_markdown("challenges.md"))
+#     st.write(read_markdown("social_impact.md"))
+#     st.write(read_markdown("bias.md"))
+
+#     col1, col2, col3, col4 = st.beta_columns([0.5,2.5,2.5,0.5])
+#     with col2:
+#         st.image("./misc/examples/female_dev_1.jpg", width=350, caption = 'German Caption: <PERSON> arbeitet an einem Computer.', use_column_width='always')
+#     with col3:
+#         st.image("./misc/examples/female_doctor.jpg", width=350, caption = 'English Caption: A portrait of <PERSON>, a doctor who specializes in health care.', use_column_width='always')
+
+#     col1, col2, col3, col4 = st.beta_columns([0.5,2.5,2.5,0.5])
+#     with col2:
+#         st.image("./misc/examples/female_doctor_1.jpg", width=350, caption = 'Spanish Caption: El Dr. <PERSON> es un estudiante de posgrado.', use_column_width='always')
+#     with col3:
+#         st.image("./misc/examples/women_cricket.jpg", width=350, caption = 'English Caption: <PERSON> of India bats against <PERSON> of Australia during the first Twenty20 match between India and Australia at Indian Bowl Stadium in New Delhi on Friday. - PTI', use_column_width='always')
+
+#     col1, col2, col3, col4 = st.beta_columns([0.5,2.5,2.5,0.5])
+#     with col2:
+#         st.image("./misc/examples/female_dev_2.jpg", width=350, caption = "French Caption: Un écran d'ordinateur avec un écran d'ordinateur ouvert.", use_column_width='always')
+#     with col3:
+#         st.image("./misc/examples/female_biker_resized.jpg", width=350, caption = 'German Caption: <PERSON> auf dem Motorrad von <PERSON>.', use_column_width='always')
+
+#     st.write(read_markdown("future_scope.md"))
+#     st.write(read_markdown("references.md"))
+#     # st.write(read_markdown("checkpoints.md"))
+#     st.write(read_markdown("acknowledgements.md"))
+
+# if state.model is None:
+#     with st.spinner("Loading model..."):
+#         state.model = load_model(checkpoints[0])
+
+# first_index = 25
+# # Init Session State
+# if state.image_file is None:
+#     state.image_file = dummy_data.loc[first_index, "image_file"]
+#     state.caption = dummy_data.loc[first_index, "caption"].strip("- ")
+#     state.lang_id = dummy_data.loc[first_index, "lang_id"]
+
+#     image_path = os.path.join("images", state.image_file)
+#     image = plt.imread(image_path)
+#     state.image = image
+
+# if st.button("Get a random example", help="Get a random example from one of the seeded examples."):
+#     sample = dummy_data.sample(1).reset_index()
+#     state.image_file = sample.loc[0, "image_file"]
+#     state.caption = sample.loc[0, "caption"].strip("- ")
+#     state.lang_id = sample.loc[0, "lang_id"]
+
+#     image_path = os.path.join("images", state.image_file)
+#     image = plt.imread(image_path)
+#     state.image = image
+
+# transformed_image = get_transformed_image(state.image)
+
+# new_col1, new_col2 = st.beta_columns([5,5])
+
+# # Display Image
+# new_col1.image(state.image, use_column_width="always")
+# # Display Reference Caption
+# with new_col1.beta_expander("Reference Caption"):
+#     st.write("**Reference Caption**: " + state.caption)
+#     st.markdown(
+#         f"""**English Translation**: {state.caption if state.lang_id == "en" else translate(state.caption, 'en')}"""
+#     )
+
+# # Select Language
+# options = list(code_to_name.keys())
+# lang_id = new_col2.selectbox(
+#     "Language",
+#     index=options.index(state.lang_id),
+#     options=options,
+#     format_func=lambda x: code_to_name[x],
+#     help="The language in which caption is to be generated."
+# )
+
+# sequence = ['']
+# if new_col2.button("Generate Caption", help="Generate a caption in the specified language."):
+#     with st.spinner("Generating Sequence..."):
+#         sequence = generate_sequence(transformed_image, lang_id, num_beams, temperature, top_p, do_sample, top_k, max_length)
+# # print(sequence)
+
+# if sequence!=['']:
+#     new_col2.write(
+#         "**Generated Caption**: "+sequence[0]
+#     )
+
+#     new_col2.write(
+#         "**English Translation**: "+ sequence[0] if lang_id=="en" else translate(sequence[0])
+#     )
+
+def main():
+    state = _get_state()
+    st.set_page_config(
+        page_title="Multilingual Image Captioning",
+        layout="wide",
+        initial_sidebar_state="collapsed",
+        page_icon="./misc/mic-logo.png",
     )
 
-# Select Language
-options = list(code_to_name.keys())
-lang_id = new_col2.selectbox(
-    "Language",
-    index=options.index(state.lang_id),
-    options=options,
-    format_func=lambda x: code_to_name[x],
-    help="The language in which caption is to be generated."
-)
-
-sequence = ['']
-if new_col2.button("Generate Caption", help="Generate a caption in the specified language."):
-    with st.spinner("Generating Sequence..."):
-        sequence = generate_sequence(transformed_image, lang_id, num_beams, temperature, top_p, do_sample, top_k, max_length)
-# print(sequence)
-
-if sequence!=['']:
-    new_col2.write(
-        "**Generated Caption**: "+sequence[0]
+    st.title("Multilingual Image Captioning")
+    st.write(
+        "[Bhavitvya Malik](https://huggingface.co/bhavitvyamalik), [Gunjan Chhablani](https://huggingface.co/gchhablani)"
     )
 
-    new_col2.write(
-        "**English Translation**: "+ sequence[0] if lang_id=="en" else translate(sequence[0])
-    )
+    st.sidebar.title("Multilingual Image Captioning")
+    logo = st.sidebar.image("./misc/mic-logo.png")
+    st.sidebar.write("Multilingual Image Captioning addresses the challenge of caption generation for an image in a multilingual setting. Here, we fuse CLIP Vision transformer into mBART50 and perform training on translated version of Conceptual-12M dataset. Please use the radio buttons below to navigate.")
+    app = MultiApp(state)
+    app.add_app("Article", article.app)
+    app.add_app("Multilingual Image Captioning", mic.app)
+    # app.add_app("Mask Filling", mlm.app)
+    app.run()
+    state.sync()
+
+if __name__ == "__main__":
+    main()

apps/article.py

@@ -0,0 +1,73 @@
+import streamlit as st
+from apps.utils import read_markdown
+# from .streamlit_tensorboard import st_tensorboard, kill_tensorboard
+from .utils import Toc
+
+def app(state=None):
+    #kill_tensorboard()
+    toc = Toc()
+    st.info("Welcome to our Multilingual Image Captioning demo. Please use the navigation sidebar to move to our demo, or scroll below to read all about our project. 🤗 In case the sidebar isn't properly rendered, please change to a smaller window size and back to full screen.")
+
+    st.header("Table of contents")
+    toc.placeholder()
+
+    toc.header("Introduction and Motivation")
+    st.write(read_markdown("intro/intro.md"))
+    toc.subheader("Novel Contributions")
+    st.write(read_markdown("intro/contributions.md"))
+
+    toc.header("Methodology")
+
+    toc.subheader("Pre-training")
+    st.write(read_markdown("pretraining/intro.md"))
+    toc.subsubheader("Dataset")
+    st.write(read_markdown("pretraining/dataset.md"))
+    _, col2, _ = st.beta_columns([1,3,1])
+    with col2:
+        st.image("./misc/Multilingual-IC.png", use_column_width='always')
+    toc.subsubheader("Model")
+    st.write(read_markdown("pretraining/model.md"))
+    # toc.subsubheader("MLM Training Logs")
+    # st.info("In case the TensorBoard logs are not displayed, please visit this link: https://huggingface.co/flax-community/multilingual-vqa-pt-ckpts/tensorboard")
+    # st_tensorboard(logdir='./logs/pretrain_logs', port=6006)
+    st.write(read_markdown("bias.md"))
+
+    _, col2, col3, _ = st.beta_columns([0.5,2.5,2.5,0.5])
+    with col2:
+        st.image("./misc/examples/female_dev_1.jpg", width=350, caption = 'German Caption: <PERSON> arbeitet an einem Computer.', use_column_width='always')
+    with col3:
+        st.image("./misc/examples/female_doctor.jpg", width=350, caption = 'English Caption: A portrait of <PERSON>, a doctor who specializes in health care.', use_column_width='always')
+
+    _, col2, col3, _ = st.beta_columns([0.5,2.5,2.5,0.5])
+    with col2:
+        st.image("./misc/examples/female_doctor_1.jpg", width=350, caption = 'Spanish Caption: El Dr. <PERSON> es un estudiante de posgrado.', use_column_width='always')
+    with col3:
+        st.image("./misc/examples/women_cricket.jpg", width=350, caption = 'English Caption: <PERSON> of India bats against <PERSON> of Australia during the first Twenty20 match between India and Australia at Indian Bowl Stadium in New Delhi on Friday. - PTI', use_column_width='always')
+
+    _, col2, col3, _ = st.beta_columns([0.5,2.5,2.5,0.5])
+    with col2:
+        st.image("./misc/examples/female_dev_2.jpg", width=350, caption = "French Caption: Un écran d'ordinateur avec un écran d'ordinateur ouvert.", use_column_width='always')
+    with col3:
+        st.image("./misc/examples/female_biker_resized.jpg", width=350, caption = 'German Caption: <PERSON> auf dem Motorrad von <PERSON>.', use_column_width='always')
+
+    toc.header("Challenges and Technical Difficulties")
+    st.write(read_markdown("challenges.md"))
+
+    toc.header("Limitations")
+    st.write(read_markdown("limitations.md"))
+
+    toc.header("Conclusion, Future Work, and Social Impact")
+    toc.subheader("Conclusion")
+    st.write(read_markdown("conclusion_future_work/conclusion.md"))
+    toc.subheader("Future Work")
+    st.write(read_markdown("conclusion_future_work/future_scope.md"))
+    toc.subheader("Social Impact")
+    st.write(read_markdown("conclusion_future_work/social_impact.md"))
+
+    toc.header("References")
+    st.write(read_markdown("references.md"))
+
+
+    toc.header("Acknowledgements")
+    st.write(read_markdown("acknowledgements.md"))
+    toc.generate()

apps/mic.py

@@ -0,0 +1,142 @@
+from .utils import get_transformed_image
+
+import streamlit as st
+import numpy as np
+import pandas as pd
+import os
+import matplotlib.pyplot as plt
+from mtranslate import translate
+from .utils import (
+    read_markdown,
+    tokenizer,
+    language_mapping,
+    code_to_name
+)
+import requests
+from PIL import Image
+from .model.flax_clip_vision_mbart.modeling_clip_vision_mbart import (
+    FlaxCLIPVisionMBartForConditionalGeneration,
+)
+from streamlit import caching
+
+
+def app(state):
+    mic_state = state
+    with st.beta_expander("Usage"):
+        st.write(read_markdown("usage.md"))
+    st.write("\n")
+    st.write(read_markdown("intro.md"))
+
+    with st.beta_expander("Generation Parameters"):
+        do_sample = st.sidebar.checkbox("Sample", value=False, help="Sample from the model instead of using beam search.")
+        top_k = st.sidebar.number_input("Top K", min_value=10, max_value=200, value=50, step=1, help="The number of highest probability vocabulary tokens to keep for top-k-filtering.")
+        num_beams = st.sidebar.number_input(label="Number of Beams", min_value=2, max_value=10, value=4, step=1, help="Number of beams to be used in beam search.")
+        temperature = st.sidebar.select_slider(label="Temperature", options = list(np.arange(0.0,1.1, step=0.1)), value=1.0, help ="The value used to module the next token probabilities.", format_func=lambda x: f"{x:.2f}")
+        top_p = st.sidebar.select_slider(label = "Top-P", options = list(np.arange(0.0,1.1, step=0.1)),value=1.0, help="Nucleus Sampling : If set to float < 1, only the most probable tokens with probabilities that add up to :obj:`top_p` or higher are kept for generation.", format_func=lambda x: f"{x:.2f}")
+
+    if st.sidebar.button("Clear All Cache"):
+        caching.clear_cache()
+
+    max_length = 64
+
+    @st.cache
+    def load_model(ckpt):
+        return FlaxCLIPVisionMBartForConditionalGeneration.from_pretrained(ckpt)
+
+    @st.cache
+    def generate_sequence(pixel_values, lang_code, num_beams, temperature, top_p, do_sample, top_k, max_length):
+        lang_code = language_mapping[lang_code]
+        output_ids = mic_state.model.generate(input_ids=pixel_values, forced_bos_token_id=tokenizer.lang_code_to_id[lang_code], max_length=max_length, num_beams=num_beams, temperature=temperature, top_p = top_p, top_k=top_k, do_sample=do_sample)
+        print(output_ids)
+        output_sequence = tokenizer.batch_decode(output_ids[0], skip_special_tokens=True, max_length=max_length)
+        return output_sequence
+
+    mic_checkpoints = ["flax-community/clip-vit-base-patch32_mbart-large-50"]  # TODO: Maybe add more checkpoints?
+    dummy_data = pd.read_csv("reference.tsv", sep="\t")
+
+    first_index = 25
+    # Init Session State
+    if mic_state.image_file is None:
+        mic_state.image_file = dummy_data.loc[first_index, "image_file"]
+        mic_state.caption = dummy_data.loc[first_index, "caption"].strip("- ")
+        mic_state.lang_id = dummy_data.loc[first_index, "lang_id"]
+
+        image_path = os.path.join("images", mic_state.image_file)
+        image = plt.imread(image_path)
+        mic_state.image = image
+
+    if mic_state.model is None:
+        # Display Top-5 Predictions
+        with st.spinner("Loading model..."):
+            mic_state.model = load_model(mic_checkpoints[0])
+
+    query1 = st.text_input(
+        "Enter a URL to an image",
+        value="http://images.cocodataset.org/val2017/000000039769.jpg",
+    )
+
+    col1, col2, col3 = st.beta_columns([2,1, 2])
+    if col1.button(
+        "Get a random example",
+        help="Get a random example from the 100 `seeded` image-text pairs.",
+    ):
+        sample = dummy_data.sample(1).reset_index()
+        mic_state.image_file = sample.loc[0, "image_file"]
+        mic_state.caption = sample.loc[0, "caption"].strip("- ")
+        mic_state.lang_id = sample.loc[0, "lang_id"]
+
+        image_path = os.path.join("images", mic_state.image_file)
+        image = plt.imread(image_path)
+        mic_state.image = image
+
+    col2.write("OR")
+
+    if col3.button("Use above URL"):
+        image_data = requests.get(query1, stream=True).raw
+        image = np.asarray(Image.open(image_data))
+        mic_state.image = image
+
+    transformed_image = get_transformed_image(mic_state.image)
+
+    new_col1, new_col2 = st.beta_columns([5,5])
+
+    # Display Image
+    new_col1.image(mic_state.image, use_column_width="always")
+    # Display Reference Caption
+    with new_col1.beta_expander("Reference Caption"):
+        st.write("**Reference Caption**: " + mic_state.caption)
+        st.markdown(
+            f"""**English Translation**: {mic_state.caption if mic_state.lang_id == "en" else translate(mic_state.caption, 'en')}"""
+        )
+
+    # Select Language
+    options = list(code_to_name.keys())
+    lang_id = new_col2.selectbox(
+        "Language",
+        index=options.index(mic_state.lang_id),
+        options=options,
+        format_func=lambda x: code_to_name[x],
+        help="The language in which caption is to be generated."
+    )
+
+    sequence = ['']
+    if new_col2.button("Generate Caption", help="Generate a caption in the specified language."):
+        with st.spinner("Generating Sequence..."):
+            sequence = generate_sequence(transformed_image, lang_id, num_beams, temperature, top_p, do_sample, top_k, max_length)
+    # print(sequence)
+
+    if sequence!=['']:
+        new_col2.write(
+            "**Generated Caption**: "+sequence[0]
+        )
+
+        new_col2.write(
+            "**English Translation**: "+ sequence[0] if lang_id=="en" else translate(sequence[0])
+        )
+
+
+
+
+# image_col, intro_col = st.beta_columns([3, 8])
+# image_col.image("./misc/mic-logo.png", use_column_width="always")
+# intro_col.write(read_markdown("intro.md"))

utils.py → apps/utils.py

@@ -4,7 +4,41 @@ import numpy as np
 from torchvision.transforms import CenterCrop, ConvertImageDtype, Normalize, Resize
 from torchvision.transforms.functional import InterpolationMode
 from PIL import Image
+import os
+import streamlit as st
+from transformers import MBart50TokenizerFast
 
+class Toc:
+    def __init__(self):
+        self._items = []
+        self._placeholder = None
+
+    def title(self, text):
+        self._markdown(text, "h1")
+
+    def header(self, text):
+        self._markdown(text, "h2", " " * 2)
+
+    def subheader(self, text):
+        self._markdown(text, "h3", " " * 4)
+
+    def subsubheader(self, text):
+        self._markdown(text, "h4", " " * 8)
+
+    def placeholder(self, sidebar=False):
+        self._placeholder = st.sidebar.empty() if sidebar else st.empty()
+
+    def generate(self):
+        if self._placeholder:
+            self._placeholder.markdown("\n".join(self._items), unsafe_allow_html=True)
+
+    def _markdown(self, text, level, space=""):
+        key = "".join(filter(str.isalnum, text)).lower()
+
+        st.markdown(f"<{level} id='{key}'>{text}</{level}>", unsafe_allow_html=True)
+        self._items.append(f"{space}* <a href='#{key}'>{text}</a>")
+
+tokenizer = MBart50TokenizerFast.from_pretrained("facebook/mbart-large-50")
 
 class Transform(torch.nn.Module):
     def __init__(self, image_size):
@@ -31,4 +65,22 @@ def get_transformed_image(image):
     if isinstance(image, np.ndarray) and image.shape[-1] == 3:
         image = image.transpose(2, 0, 1)
         image = torch.tensor(image)
-    return transform(image).unsqueeze(0).permute(0, 2, 3, 1).numpy()
+    return transform(image).unsqueeze(0).permute(0, 2, 3, 1).numpy()
+
+def read_markdown(path, parent="./sections/"):
+    with open(os.path.join(parent, path)) as f:
+        return f.read()
+
+language_mapping = {
+    "en": "en_XX",
+    "de": "de_DE",
+    "fr": "fr_XX",
+    "es": "es_XX"
+}
+
+code_to_name = {
+    "en": "English",
+    "fr": "French",
+    "de": "German",
+    "es": "Spanish",
+}

multiapp.py

@@ -0,0 +1,16 @@
+import streamlit as st
+
+class MultiApp:
+    def __init__(self, state):
+        self.apps = []
+        self.state = state
+
+    def add_app(self, title, func):
+        self.apps.append({"title": title, "function": func})
+
+    def run(self):
+        st.sidebar.header("Go To:")
+        app = st.sidebar.radio(
+            "", self.apps, format_func=lambda app: app["title"]
+        )
+        app["function"](self.state)

sections/acknowledgements.md

@@ -1,4 +1,3 @@
-## Acknowledgements
 We'd like to thank [Abheesht Sharma](https://huggingface.co/abheesht) for helping in the discussions in the initial phases. [Luke Melas](https://github.com/lukemelas) helped us get the cleaned CC-12M data on our TPU-VMs and we are very grateful to him.
 
 This project would not be possible without the help of [Patrick](https://huggingface.co/patrickvonplaten) and [Suraj](https://huggingface.co/valhalla) who met with us and helped us review our approach and guided us throughout the project. We especially thank Patrick for going out of the way and allowing us extra TPU time so that we could work on this project.

sections/challenges.md

@@ -1,4 +1,3 @@
-## Challenges and Technical Difficulties
 Training image captioning that too multilingual was a difficult task and we faced challenges at almost every point of this process.
 
 - Dataset: Our initial plan was to translate Conceptual 12M using mTranslate or Yandex but they turned out to be too slow even with multiprocessing. Not having proper translation could lead to poor performance of the trained image-caption model. We translated the whole dataset using MBart50 for all languages which took around 3-4 days. Further on, we realised that mBART captions were not that good and model was not converging because of that which lead us to re-translate our captions with [Marian](https://huggingface.co/transformers/model_doc/marian.html)

sections/{future_scope.md → conclusion_future_work/future_scope.md}

@@ -1,4 +1,3 @@
-## Future scope of work
 We hope to improve this project in the future by using:
 - Better options for data translation: Translation has a very huge impact on how the end model would perform. Better translators (for e.g. Google Translate API) and language specific seq2seq models for translation are able to generate better data, both for high-resource and low-resource languages.
 - Accessibility: Make model deployable on hand-held devices to make it more accessible. Currently, our model is too large to fit on mobile/edge devices because of which not many will be able to access it. However, our final goal is ensure everyone can access it without any computation barriers. We got to know that JAX has an experimental converter `jax2tf`to convert JAX functions to TF. Hopefully we'll be able to support TFLite for our model as well in future.

sections/{social_impact.md → conclusion_future_work/social_impact.md}

@@ -1,4 +1,3 @@
-## Social Impact
 Being able to automatically describe the content of an image using properly formed sentences in any language is a challenging task, but it could have great impact by helping visually impaired people better understand their surroundings.
 
 Our initial plan was to include 4 high-resource and 4 low-resource languages (Marathi, Bengali, Urdu, Telegu) in our training data. However, the existing translations do not perform as well and we would have received poor labels, not to mention, with a longer training time.

sections/pretraining.md

@@ -1,4 +1,3 @@
-### Pretraining
 We follow an encoder-decoder approach for image captioning, where the image encoder is the CLIP Vision model (a ViT transformer). The pre-training task is image-to-text generation. We take the input tokens and shift them using an `<eos>` token towards right in order to create the inputs for our model, while the original input tokens become labels. The model is trained on the dataset. in an end-to-end fashion.
 
 **Dataset**

sections/pretraining/dataset.md

@@ -0,0 +1 @@
+The dataset we use for pre-training is a cleaned version of Conceptual 12M. The dataset is downloaded and then broken images are removed which gives us about 10M images. To save time, we use 2.5M of these image-text pairs. Then we use the MarianMT `Helsinki-NLP/opus-mt-{src}-{tgt}` checkpoint to translate the dataset into four different languages - English, French, German, and Spanish, keeping approximately 2.5M examples of each language.

sections/pretraining/intro.md

@@ -0,0 +1 @@
+We follow an encoder-decoder approach for image captioning, where the image encoder is the CLIP Vision model (a ViT transformer). The pre-training task is image-to-text generation. We take the input tokens and shift them using an `<eos>` token towards right in order to create the inputs for our model, while the original input tokens become labels. The model is trained on the dataset. in an end-to-end fashion.

sections/pretraining/model.md

@@ -0,0 +1,10 @@
+The model is shown in the image above. We create a custom model in Flax which integerates the CLIP Vision model as an encoder inside mBART model. We also use custom configs and modules in order to accomodate for these changes, and allow loading from mBART and CLIP Vision checkpoints. The image is fed to the CLIP Vision encoder and the shifted token ids are  fed to the mBART decoder. We use the `facebook/mbart-large-50` and `openai/clip-vit-base-patch32` checkpoints for mBART and CLIP Vision models, respectively. All our code is available on [GitHub](https://github.com/gchhablani/multilingual-image-captioning).
+
+Our model reached **eval loss of ~2.6** around ~70K steps. Here are the BLEU scores (out of 1) for different languages:
+
+|Language                  |BLEU-1|BLEU-2|BLEU-3|BLEU-4|
+|--------------------------|------|------|------|------|
+|English                   | 0.13083| 0.08887| 0.06681 | 0.04899|
+|Spanish                   | 0.15981| 0.09858| 0.06918| 0.04776|
+|German                    | 0.14234| 0.09817| 0.07405| 0.0515|
+|French                    | 0.13021| 0.08862| 0.06598| 0.04647|

sections/references.md

@@ -1,5 +1,3 @@
-## References
-
 ```
 @inproceedings{wolf-etal-2020-transformers,
     title = "Transformers: State-of-the-Art Natural Language Processing",