File size: 3,795 Bytes
44c11f2
 
c399665
ba57ea8
ffe19d9
 
 
53ddc87
f54892c
b3c9da2
b929de0
919efff
66b1416
b3c9da2
0b8741b
b3c9da2
d63921e
3bd4b4e
 
 
 
b3c9da2
919efff
 
b3c9da2
919efff
b3c9da2
919efff
 
29e00f0
44c11f2
b3c9da2
44c11f2
8527e35
919efff
8527e35
919efff
8527e35
76f1340
b929de0
76f1340
 
919efff
 
b3c9da2
8527e35
919efff
8527e35
919efff
8527e35
76f1340
b929de0
76f1340
 
 
919efff
b3c9da2
44c11f2
919efff
0b8741b
287b7cd
919efff
7f8d82b
ffe19d9
 
 
 
 
53ddc87
919efff
b3c9da2
0b8741b
 
 
 
 
919efff
b3c9da2
53ddc87
b3c9da2
53ddc87
919efff
b3c9da2
919efff
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
import streamlit as st
from apps.utils import read_markdown
from .streamlit_tensorboard import st_tensorboard, kill_tensorboard
from .utils import Toc


    
def app(state=None):
    #kill_tensorboard()
    toc = Toc()
    st.info("Welcome to our Multilingual-VQA demo. Please use the navigation sidebar to move to our demo, or scroll below to read all about our project. 🤗 In case the sidebar isn't properly rendered, please change to a smaller window size and back to full screen.")
    
    st.header("Table of Contents")
    toc.placeholder()

    toc.header("Introduction and Motivation")
    st.info("**News**: Two days back, a paper using CLIP-Vision and BERT has been posted on arXiv! The paper uses LXMERT objectives and achieves 80% on the English VQAv2 dataset. It would be interesting to see how it performs on our multilingual dataset. Check it out here: https://arxiv.org/pdf/2107.06383.pdf")
    st.write(read_markdown("intro/intro_part_1.md"))
    with st.beta_expander("FasterRCNN Approach"):
        st.write(read_markdown("intro/faster_rcnn_approach.md"))
    st.write(read_markdown("intro/intro_part_2.md"))
    toc.subheader("Novel Contributions")
    st.write(read_markdown("intro/contributions.md"))
    
    toc.header("Methodology")

    toc.subheader("Pre-training")
    st.write(read_markdown("pretraining/intro.md"))
    # col1, col2 = st.beta_columns([5,5])
    st.image(
        "./misc/article/Multilingual-VQA.png",
        caption="Masked LM model for Image-text Pre-training.",
    )
    toc.subsubheader("MLM Dataset")
    st.write(read_markdown("pretraining/data.md"))
    toc.subsubheader("MLM Model")
    st.write(read_markdown("pretraining/model.md"))
    toc.subsubheader("MLM Training Logs")
    st.write("Click on the expandable region to see the TensorBoard logs.")
    st.info("In case the TensorBoard logs are not displayed, please visit this link: https://huggingface.co/flax-community/multilingual-vqa-pt-ckpts/tensorboard")
    with st.beta_expander("MLM TensorBoard Logs"):
        st_tensorboard(logdir='./logs/pretrain_logs', port=6006)
    
    
    toc.subheader("Finetuning")
    toc.subsubheader("VQA Dataset")
    st.write(read_markdown("finetuning/data.md"))
    toc.subsubheader("VQA Model")
    st.write(read_markdown("finetuning/model.md"))
    toc.subsubheader("VQA Training Logs")
    st.write("Click on the expandable region to see the TensorBoard logs.")
    st.info("In case the TensorBoard logs are not displayed, please visit this link: https://huggingface.co/flax-community/multilingual-vqa-pt-60k-ft/tensorboard")
    with st.beta_expander("VQA TensorBoard Logs"):
        st_tensorboard(logdir='./logs/finetune_logs', port=6007)
    
    
    toc.header("Challenges and Technical Difficulties")
    st.write(read_markdown("challenges.md"))
    
    toc.header("Limitations and Bias")
    st.write(read_markdown("limitations.md"))
    
    toc.header("Conclusion, Future Work, and Social Impact")
    # toc.subheader("Conclusion")
    # st.write(read_markdown("conclusion_future_work/conclusion.md"))
    # toc.subheader("Future Work")
    # st.write(read_markdown("conclusion_future_work/future_work.md"))
    # toc.subheader("Social Impact")
    st.write(read_markdown("conclusion_future_work/social_impact.md"))
    
    toc.header("References")
    toc.subheader("Papers")
    st.write(read_markdown("references/papers.md"))
    toc.subheader("Useful Links")
    st.write(read_markdown("references/useful_links.md"))


    toc.header("Checkpoints")
    st.write(read_markdown("checkpoints/checkpoints.md"))
    toc.subheader("Other Checkpoints")
    st.write(read_markdown("checkpoints/other_checkpoints.md"))
    
    toc.header("Acknowledgements")
    st.write(read_markdown("acknowledgements.md"))
    toc.generate()