Upload 164 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+repo_imgs/sample_1.gif filter=lfs diff=lfs merge=lfs -text
+repo_imgs/sample_2.gif filter=lfs diff=lfs merge=lfs -text
+repo_imgs/sample_3.gif filter=lfs diff=lfs merge=lfs -text

LICENSE.md

@@ -0,0 +1,14 @@
+BSD 3-Clause License
+
+Copyright 2023 Deyao Zhu
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE_Lavis.md

@@ -0,0 +1,14 @@
+BSD 3-Clause License
+
+Copyright (c) 2022 Salesforce, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

datasets/training_datasets/video_text_data/video_instruct_100/download_script.py

@@ -0,0 +1,94 @@
+import json 
+from tqdm import tqdm
+from pytubefix import YouTube
+
+import xml.etree.ElementTree as ET
+import os
+
+with open ('VideoInstruct100K.json','r') as f :
+    data=json.load(f)
+
+# Usage
+existed_video_id={}
+for video_name in os.listdir('videos'):
+    video_id = video_name.split('.')[0]
+    existed_video_id[video_id]=True 
+
+
+
+def download_video_with_subtitles(video_id):
+    # Create a YouTube object.
+    yt = YouTube(f'https://www.youtube.com/watch?v={video_id}')
+
+    video_filename = f"{video_id}.mp4"
+    video_downloaded=False
+    try :
+        # Get the video stream with the highest resolution and download the video.
+        stream = yt.streams.get_highest_resolution()
+        stream.download(output_path='videos', filename=video_filename)
+        video_downloaded=True
+    except Exception as e:
+        print(f"Error downloading video {video_id}: {str(e)}")
+        video_downloaded=False
+    if not video_downloaded:
+        return False,False
+
+    # Get the video's available captions (subtitles).
+    captions = yt.captions.all()
+
+    # Download the captions if available in xml format.
+    caption_downloaded = False
+    for caption in captions:
+        caption_code = caption.code
+        # select only english captions
+        if 'en' in caption_code:
+            caption.download(title=f"{video_id}", output_path='subtitles_xml',srt=False)
+            caption_downloaded = True
+    return video_downloaded,caption_downloaded
+def convert_xml_vtt(xml_path, vtt_path):
+    # Parse the XML subtitle file
+    tree = ET.parse(xml_path)
+    root = tree.getroot()
+
+    # Initialize a list to store VTT subtitle entries
+    vtt_subtitle = []
+
+    # Function to convert time in milliseconds to WebVTT format
+    def ms_to_vtt_time(milliseconds):
+        seconds, milliseconds = divmod(milliseconds, 1000)
+        minutes, seconds = divmod(seconds, 60)
+        return f"{minutes:02d}:{seconds:02d}.{milliseconds:03d}"
+
+    # Iterate through subtitle elements
+    toggle = True
+    for p in root.findall(".//p"):
+        if toggle:
+            start_time = int(p.get("t"))
+            subtitle_text = " ".join(s.text.strip() for s in p.findall(".//s"))
+        # duration = int(p.get("d")) if p.get("d") is not None else 0
+        if not toggle:
+            end_time = int(p.get("t"))
+            # Format and append the VTT entry to the list
+            vtt_subtitle.append(f"{ms_to_vtt_time(start_time)} --> {ms_to_vtt_time(end_time)}\n{subtitle_text}\n")
+        toggle = not toggle
+    # Join the VTT entries into a single string
+    vtt_content = "WEBVTT\n\n" + "\n".join(vtt_subtitle)
+
+    # Save the VTT content to a file
+    with open(vtt_path, "w", encoding="utf-8") as vtt_file:
+        vtt_file.write(vtt_content)
+import os
+os.makedirs('videos', exist_ok=True)
+os.makedirs('subtitles_vtt', exist_ok=True)
+os.makedirs('subtitles_xml', exist_ok=True)
+for video_path in tqdm(data,desc='Downloading videos') :
+    video_id=video_path.split('/')[-1].split('.')[0]
+    if existed_video_id.get(video_id,False):
+        continue
+    video_downloaded,caption_downloaded=download_video_with_subtitles(video_id)
+    if caption_downloaded:
+        # convert xml to vtt
+        xml_file_path=f'subtitles_xml/{video_id} (a.en).xml'
+        convert_xml_vtt(xml_file_path,f'subtitles_vtt/{video_id}.vtt')
+        
+    

demo_job.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=video_demo_llama2
+#SBATCH --output=video_demo_llama2.out
+#SBATCH --error=video_demo_llama2.err
+#SBATCH --time=0-10:30:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+#  Choose the model to test 
+# Mistral 
+# ckpt="checkpoints/video_mistral_checkpoint_last.pth"
+# config="test_configs/mistral_test_config.yaml"
+
+# Llama2
+ckpt="checkpoints/video_llama_checkpoint_last.pth"
+config="test_configs/llama2_test_config.yaml"
+
+
+python minigpt4_video_demo.py --cfg-path $config --ckpt $ckpt

environment.yml

@@ -0,0 +1,331 @@
+name: minigpt4_video_test_v100
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - archspec=0.2.2=pyhd8ed1ab_0
+  - boltons=23.1.1=pyhd8ed1ab_0
+  - brotli-python=1.1.0=py39h3d6467e_1
+  - bzip2=1.0.8=hd590300_5
+  - c-ares=1.25.0=hd590300_0
+  - ca-certificates=2024.2.2=hbcca054_0
+  - certifi=2024.2.2=pyhd8ed1ab_0
+  - cffi=1.16.0=py39h7a31438_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - conda=23.11.0=py39hf3d152e_1
+  - conda-libmamba-solver=23.12.0=pyhd8ed1ab_0
+  - conda-package-handling=2.2.0=pyh38be061_0
+  - conda-package-streaming=0.9.0=pyhd8ed1ab_0
+  - cudatoolkit=11.8.0=h4ba93d1_12
+  - cudatoolkit-dev=11.7.0=h1de0b5d_6
+  - distro=1.9.0=pyhd8ed1ab_0
+  - faiss=1.7.4=py39cuda112h460e57a_0_cuda
+  - fmt=10.1.1=h00ab1b0_1
+  - freetype=2.12.1=h267a509_2
+  - gmp=6.1.2=hf484d3e_1000
+  - gnutls=3.5.19=h2a4e5f8_1
+  - icu=73.2=h59595ed_0
+  - idna=3.6=pyhd8ed1ab_0
+  - jsonpatch=1.33=pyhd8ed1ab_0
+  - jsonpointer=2.4=py39hf3d152e_3
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.21.2=h659d440_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libarchive=3.7.2=h2aa1ff5_1
+  - libblas=3.9.0=20_linux64_openblas
+  - libcblas=3.9.0=20_linux64_openblas
+  - libcurl=8.5.0=hca28451_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=hd590300_2
+  - libfaiss=1.7.4=cuda112hb18a002_0_cuda
+  - libfaiss-avx2=1.7.4=cuda112h1234567_0_cuda
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=13.2.0=h807b86a_3
+  - libgfortran-ng=13.2.0=h69a702a_3
+  - libgfortran5=13.2.0=ha4646dd_3
+  - libgomp=13.2.0=h807b86a_3
+  - libiconv=1.17=hd590300_2
+  - liblapack=3.9.0=20_linux64_openblas
+  - libmamba=1.5.6=had39da4_0
+  - libmambapy=1.5.6=py39h10defb6_0
+  - libnghttp2=1.58.0=h47da74e_1
+  - libnsl=2.0.1=hd590300_0
+  - libopenblas=0.3.25=pthreads_h413a1c8_0
+  - libpng=1.6.39=h753d276_0
+  - libsolv=0.7.27=hfc55251_0
+  - libsqlite=3.44.2=h2797004_0
+  - libssh2=1.11.0=h0841786_0
+  - libstdcxx-ng=13.2.0=h7e041cc_3
+  - libuuid=2.38.1=h0b41bf4_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxml2=2.12.3=h232c23b_0
+  - libzlib=1.2.13=hd590300_5
+  - lz4-c=1.9.4=hcb278e6_0
+  - lzo=2.10=h516909a_1000
+  - menuinst=2.0.1=py39hf3d152e_0
+  - ncurses=6.4=h59595ed_2
+  - nettle=3.3=0
+  - numpy=1.26.3=py39h474f0d3_0
+  - openh264=1.8.0=hdbcaa40_1000
+  - openssl=3.2.1=hd590300_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - pip=23.3.2=pyhd8ed1ab_0
+  - platformdirs=4.1.0=pyhd8ed1ab_0
+  - pluggy=1.3.0=pyhd8ed1ab_0
+  - pybind11-abi=4=hd8ed1ab_3
+  - pycosat=0.6.6=py39hd1e30aa_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.9.18=h0755675_1_cpython
+  - python_abi=3.9=4_cp39
+  - readline=8.2=h8228510_1
+  - reproc=14.2.4.post0=hd590300_1
+  - reproc-cpp=14.2.4.post0=h59595ed_1
+  - requests=2.31.0=pyhd8ed1ab_0
+  - ruamel.yaml=0.18.5=py39hd1e30aa_0
+  - ruamel.yaml.clib=0.2.7=py39hd1e30aa_2
+  - tk=8.6.13=noxft_h4845f30_101
+  - tqdm=4.66.1=pyhd8ed1ab_0
+  - urllib3=2.1.0=pyhd8ed1ab_0
+  - wheel=0.42.0=pyhd8ed1ab_0
+  - x264=1!152.20180717=h14c3975_1001
+  - xz=5.2.6=h166bdaf_0
+  - yaml-cpp=0.8.0=h59595ed_0
+  - zlib=1.2.13=hd590300_5
+  - zstandard=0.22.0=py39h6e5214e_0
+  - zstd=1.5.5=hfc55251_0
+  - pip:
+      - accelerate==0.25.0
+      - aiofiles==23.2.1
+      - aiohttp==3.9.1
+      - aiosignal==1.3.1
+      - altair==5.2.0
+      - annotated-types==0.6.0
+      - antlr4-python3-runtime==4.9.3
+      - anyio==4.2.0
+      - appdirs==1.4.4
+      - asgiref==3.7.2
+      - async-timeout==4.0.3
+      - attrs==23.2.0
+      - backoff==2.2.1
+      - bcrypt==4.1.2
+      - beautifulsoup4==4.12.2
+      - bitarray==2.9.2
+      - bitsandbytes==0.42.0
+      - bleach==6.1.0
+      - blinker==1.7.0
+      - braceexpand==0.1.7
+      - build==1.0.3
+      - cachetools==5.3.2
+      - chardet==5.2.0
+      - chroma-hnswlib==0.7.3
+      - chromadb==0.4.22
+      - click==8.1.7
+      - cmake==3.25.0
+      - colbert-ai==0.2.18
+      - coloredlogs==15.0.1
+      - contourpy==1.2.0
+      - cycler==0.12.1
+      - datasets==2.17.0
+      - decorator==4.4.2
+      - decord==0.6.0
+      - deprecated==1.2.14
+      - dill==0.3.8
+      - docker-pycreds==0.4.0
+      - docopt==0.6.2
+      - einops==0.7.0
+      - exceptiongroup==1.2.0
+      - faiss-gpu==1.7.2
+      - fastapi==0.108.0
+      - ffmpeg==1.4
+      - ffmpeg-python==0.2.0
+      - ffmpy==0.3.1
+      - filelock==3.13.1
+      - flash-attn==2.5.4
+      - flask==3.0.2
+      - flatbuffers==23.5.26
+      - fonttools==4.47.0
+      - frozenlist==1.4.1
+      - fsspec==2023.10.0
+      - ftfy==6.1.3
+      - future==0.18.3
+      - gdown==4.7.1
+      - git-python==1.0.3
+      - gitdb==4.0.11
+      - gitpython==3.1.40
+      - google-auth==2.26.1
+      - googleapis-common-protos==1.62.0
+      - gradio
+      - gradio-client
+      - h11==0.14.0
+      - h5py==3.10.0
+      - httpcore==1.0.2
+      - httptools==0.6.1
+      - httpx==0.26.0
+      - huggingface-hub==0.21.1
+      - humanfriendly==10.0
+      - imageio==2.33.1
+      - imageio-ffmpeg==0.4.9
+      - importlib-metadata==6.11.0
+      - importlib-resources==6.1.1
+      - inquirerpy==0.3.4
+      - iopath==0.1.10
+      - itsdangerous==2.1.2
+      - jinja2==3.1.2
+      - joblib==1.3.2
+      - jsonschema==4.20.0
+      - jsonschema-specifications==2023.12.1
+      - kaggle==1.6.0
+      - kiwisolver==1.4.5
+      - kubernetes==29.0.0
+      - lazy-loader==0.3
+      - lit==15.0.7
+      - llvmlite==0.41.1
+      - markdown-it-py==3.0.0
+      - matplotlib==3.8.2
+      - mdurl==0.1.2
+      - mmh3==4.1.0
+      - monotonic==1.6
+      - more-itertools==10.1.0
+      - moviepy==1.0.3
+      - mpmath==1.3.0
+      - multidict==6.0.4
+      - multiprocess==0.70.16
+      - mutagen==1.47.0
+      - networkx==3.2.1
+      - ninja==1.11.1.1
+      - nltk==3.8.1
+      - numba==0.58.1
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-nccl-cu12==2.18.1
+      - nvidia-nvjitlink-cu12==12.3.101
+      - nvidia-nvtx-cu12==12.1.105
+      - omegaconf==2.3.0
+      - onnxruntime==1.16.3
+      - openai==0.28.0
+      - openai-whisper==20231117
+      - opencv-python==4.7.0.72
+      - opentelemetry-api==1.22.0
+      - opentelemetry-exporter-otlp-proto-common==1.22.0
+      - opentelemetry-exporter-otlp-proto-grpc==1.22.0
+      - opentelemetry-instrumentation==0.43b0
+      - opentelemetry-instrumentation-asgi==0.43b0
+      - opentelemetry-instrumentation-fastapi==0.43b0
+      - opentelemetry-proto==1.22.0
+      - opentelemetry-sdk==1.22.0
+      - opentelemetry-semantic-conventions==0.43b0
+      - opentelemetry-util-http==0.43b0
+      - orjson==3.9.10
+      - overrides==7.4.0
+      - pandas==2.0.0
+      - pathtools==0.1.2
+      - peft==0.2.0
+      - pfzy==0.3.4
+      - pillow==10.2.0
+      - plotly==5.18.0
+      - portalocker==2.8.2
+      - posthog==3.3.0
+      - proglog==0.1.10
+      - progressbar2==4.3.2
+      - prompt-toolkit==3.0.43
+      - protobuf==4.25.1
+      - psutil==5.9.7
+      - pulsar-client==3.4.0
+      - pyarrow==15.0.0
+      - pyarrow-hotfix==0.6
+      - pyasn1==0.5.1
+      - pyasn1-modules==0.3.0
+      - pycocoevalcap==1.2
+      - pycocotools==2.0.6
+      - pycryptodomex==3.19.1
+      - pydantic==2.5.3
+      - pydantic-core==2.14.6
+      - pydub==0.25.1
+      - pygments==2.17.2
+      - pyparsing==3.1.1
+      - pypika==0.48.9
+      - pyproject-hooks==1.0.0
+      - pysrt==1.1.2
+      - python-dateutil==2.8.2
+      - python-dotenv==1.0.0
+      - python-multipart==0.0.6
+      - python-slugify==8.0.1
+      - python-utils==3.8.1
+      - pytubefix
+      - pytz==2023.3.post1
+      - pyyaml==6.0.1
+      - referencing==0.32.0
+      - regex==2023.12.25
+      - rich==13.7.0
+      - rouge==1.0.1
+      - rpds-py==0.16.2
+      - rsa==4.9
+      - safetensors==0.4.1
+      - scikit-image==0.22.0
+      - scikit-learn==1.3.2
+      - scipy==1.11.4
+      - seaborn==0.13.1
+      - semantic-version==2.10.0
+      - sentence-transformers==2.2.2
+      - sentencepiece==0.1.97
+      - sentry-sdk==1.39.1
+      - setproctitle==1.3.3
+      - setuptools==69.0.3
+      - shellingham==1.5.4
+      - six==1.16.0
+      - smmap==5.0.1
+      - sniffio==1.3.0
+      - soundfile==0.12.1
+      - soupsieve==2.5
+      - starlette==0.32.0.post1
+      - sympy==1.12
+      - tenacity==8.2.3
+      - text-unidecode==1.3
+      - threadpoolctl==3.2.0
+      - tifffile==2023.12.9
+      - tiktoken==0.5.2
+      - timm==0.6.13
+      - tokenizers==0.15.2
+      - tomli==2.0.1
+      - tomlkit==0.12.0
+      - toolz==0.12.0
+      - torch==2.0.1
+      - torchaudio==2.0.2
+      - torchvision==0.15.2
+      - transformers==4.37.2
+      - triton==2.0.0
+      - typer==0.9.0
+      - typing-extensions==4.9.0
+      - tzdata==2023.4
+      - ujson==5.9.0
+      - uvicorn==0.25.0
+      - uvloop==0.19.0
+      - visual-genome==1.1.1
+      - wandb==0.14.2
+      - watchfiles==0.21.0
+      - wcwidth==0.2.13
+      - webdataset==0.2.48
+      - webencodings==0.5.1
+      - websocket-client==1.7.0
+      - websockets
+      - webvtt-py==0.4.6
+      - wrapt==1.16.0
+      - xxhash==3.4.1
+      - yarl==1.9.4
+      - youtube-dl==2021.12.17
+      - yt-dlp
+      - zipp

eval_video.py

@@ -0,0 +1,221 @@
+import os
+import json
+from tqdm import tqdm
+from torch.utils.data import DataLoader
+from minigpt4.common.eval_utils import prepare_texts, init_model, eval_parser
+from minigpt4.conversation.conversation import CONV_VISION
+from minigpt4.processors.blip_processors import Blip2ImageTrainProcessor,BlipCaptionProcessor
+from minigpt4.datasets.datasets.video_datasets import VideoChatGPTEvalDataset,VideoChatGPTEval_consistancy,Video_validation_Dataset,TVQAEVAL,TVQAEVAL_Long
+
+parser = eval_parser()
+parser.add_argument("--dataset", type=str, default='msvd', help="dataset to evaluate")
+parser.add_argument("--add_subtitles",action='store_true',help="whether to add subtitles to the video")
+parser.add_argument("--name", type=str, default='3_datasets', help="evaluation name")
+parser.add_argument("--batch_size", type=int, default=1, help="batch size")
+parser.add_argument("--start", type=int, default=0, help="start from video number")
+parser.add_argument("--end", type=int, default=10000000, help="end at video number")
+args = parser.parse_args()
+
+print(args.ckpt)
+print(args.name)
+print(args.cfg_path)
+if "test_configs/mistral_test_config.yaml" == args.cfg_path: 
+    llm_name="mistral"
+else:   
+    llm_name="llama2"
+print("using captions",args.add_subtitles)
+
+model, vis_processor = init_model(args)
+conv_temp = CONV_VISION.copy()
+conv_temp.system = ""
+if args.dataset == 'video_chatgpt_generic':
+    ann_path="datasets/evaluation_datasets/videochatgpt_benchmark/generic_qa.json"
+    videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Test_Videos"
+    subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles"
+    videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/benchmark/generic"
+    annotations_keys=['Q','A','video_name']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'video_chatgpt_temporal':
+    ann_path="datasets/evaluation_datasets/videochatgpt_benchmark/temporal_qa.json"
+    videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Test_Videos"
+    subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles"
+    videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/benchmark/temporal"
+    annotations_keys=['Q','A','video_name']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'video_chatgpt_consistency':
+    ann_path="datasets/evaluation_datasets/videochatgpt_benchmark/consistency_qa.json"
+    videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Test_Videos"
+    subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles"
+    annotations_keys=[['Q1','Q2'],'A','video_name']
+    data = VideoChatGPTEval_consistancy(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name)
+    
+elif args.dataset == 'msrvtt':
+    ann_path="datasets/evaluation_datasets/msrvtt/val_qa_edited.json"
+    videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/MSRVTT/videos/all"
+    subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles"
+    videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/msrvtt"
+    annotations_keys=['question','answer','video_id']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name)
+
+elif args.dataset == 'msvd':
+    ann_path="datasets/evaluation_datasets/msvd/val_qa_edited.json"
+    videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/MSVD-QA/videos"
+    subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles"
+    videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/msvd"
+    annotations_keys=['question','answer','video_id']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'activitynet':
+    ann_path="datasets/evaluation_datasets/activityNet/test_qa.json"
+    videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Activity_net/Activity_net_videos"
+    subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles/"
+    videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/activity_net"
+    annotations_keys=['question','answer','video_id']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'tgif':
+    ann_path="datasets/evaluation_datasets/tgif/Test_frameqa_question.json"
+    videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/TGIF/mp4s"
+    subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles"
+    videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/tgif"
+    annotations_keys=['question','answer','gif_name']
+    # annotations_keys=['question','description','gif_name']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=False,llm_name=llm_name)
+elif args.dataset == 'tvqa':
+    # TVQA dataset
+    ann_path="datasets/evaluation_datasets/tvqa_short/tvqa_val.json"
+    videos_path= "/ibex/project/c2090/datasets/TVR_dataset/videos/video_files/frames_hq/"
+    subtitles_path="/ibex/project/c2090/datasets/TVR_dataset/TVRetrieval/data/tvqa_preprocessed_subtitles.json"
+    videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/tvqa"
+    data = TVQAEVAL(vis_processor, videos_path, ann_path,subtitles_path,videos_features_path,add_subtitles=args.add_subtitles,llm_name=llm_name)
+
+eval_dataloader = DataLoader(data, batch_size=args.batch_size, shuffle=False)
+
+minigpt4_predict = []
+sub="subtitles" if args.add_subtitles else "no_subtitles"
+if args.start == 0 and args.end == 10000000:
+    save_path = f'results/{args.name}_{args.dataset}_{sub}.json'
+else:
+    print("start from video number",args.start)
+    print("end at video number",args.end)
+    save_path = f'results/{args.name}_{args.dataset}_{sub}_{args.start}_{args.end}.json'
+
+os.makedirs("results", exist_ok=True)
+c=0
+pred_result = {}
+gt_result = {}
+if args.dataset == 'video_chatgpt_consistency':
+    for images, texts_1,texts_2, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts_q1 = prepare_texts(texts_1, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            texts_q2 = prepare_texts(texts_2, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers_q1 = model.generate(images, texts_q1, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            models_answers_q2 = model.generate(images, texts_q2, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer_q1,model_answer_q2, gt_answer,text_q1,text_q2 in zip(videos_ids,models_answers_q1,models_answers_q2, gt_answers,texts_q1,texts_q2):
+                result = dict()
+                result['video_name'] = video_id
+                result['Q1'] = text_q1.split('\n')[-1].replace('[/INST]','')
+                result['Q2'] = text_q2.split('\n')[-1].replace('[/INST]','')
+                result['A'] = gt_answer
+                result['pred1'] = model_answer_q1
+                result['pred2'] = model_answer_q2
+                pred_result[video_id] = [model_answer_q1,model_answer_q2]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+
+elif args.dataset == 'tvr':
+    for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts = prepare_texts(texts, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers = model.generate(images, texts, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer, gt_answer,text in zip(videos_ids,models_answers, gt_answers,texts):
+                result = dict()
+                result['video_name'] = video_id
+                result['Q'] = text.split('\n')[-1].replace('[/INST]','')
+                result['A'] = gt_answer
+                result['pred'] = model_answer
+                pred_result[video_id] = [model_answer]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+elif args.dataset == 'ego_schema' or args.dataset == 'tvqa' or args.dataset == 'tvqa_long_videos':
+    for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts = prepare_texts(texts, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers = model.generate(images, texts, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer, gt_answer,text in zip(videos_ids,models_answers, gt_answers,texts):
+                result = dict()
+                result['video_name'] = video_id
+                if args.dataset == 'tvqa_long_videos':
+                    result['Q'] = text.split('\n\n')[1:]
+                else:
+                    result['Q'] = text.split('\n')[1:]
+                result['A'] = gt_answer
+                result['pred'] = model_answer
+                pred_result[video_id] = [model_answer]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+else:
+    for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts = prepare_texts(texts, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers = model.generate(images, texts, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer, gt_answer,text in zip(videos_ids,models_answers, gt_answers,texts):
+                result = dict()
+                result['video_name'] = video_id
+                result['Q'] = text.split('\n')[-1].replace('[/INST]','')
+                result['A'] = gt_answer
+                result['pred'] = model_answer
+                pred_result[video_id] = [model_answer]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+
+with open(save_path, 'w') as f:
+    json.dump(minigpt4_predict, f)
+print("saved results to",save_path)
+# save results
+# bleu_save_path = f'results/{args.name}_{args.dataset}_bleu.json'
+# cider_save_path = f'results/{args.name}_{args.dataset}_cider.json'
+# chatgpt_eval_save_path = f'results/{args.name}_{args.dataset}_chatgpt_eval.json'
+# bleu_results=eval_bleu(minigpt4_predict)
+# with open(bleu_save_path, 'w') as f:
+#     json.dump(bleu_results, f)
+# print("bleu_results",bleu_results)
+# cider_results=eval_cider(pred_result,gt_result)
+# with open(cider_save_path, 'w') as f:
+#     json.dump(cider_results, f)
+# print("mean_cider_scores:",cider_results['mean_cider_scores'])
+
+# chatgpt_results=chat_gpt_eval(pred_result,gt_result)
+
+# with open(chatgpt_eval_save_path, 'w') as f:
+#     json.dump(chatgpt_results, f)
+# print("avg_chatgpt_score",chatgpt_results['avg_chatgpt_score'])
+# print(chatgpt_results)
+
+

jobs_video/eval/choose_best_ckpt/choose_best_ckpt.py

@@ -0,0 +1,14 @@
+import os 
+import shutil 
+ckpt_dir = 'ckpt_dir'
+print(f'number of ckpts: {len(os.listdir(ckpt_dir))}')
+for ckpt in sorted(os.listdir(ckpt_dir)):
+    if not ckpt.endswith('.pth'):
+        continue
+    ckpt_path = os.path.join(ckpt_dir,ckpt)
+    job_name="cmd_webvid_video_instruct_"+ckpt.split(".")[0]
+    # submit a job with this ckpt file
+    os.system(f'sbatch ./evalualtion_ckpt.sh {ckpt_path} {job_name}')
+    # print(f'sbatch ./evalualtion_ckpt.sh {ckpt_path} {job_name}')
+    # print(f'job {job_name} submitted')
+    # break

jobs_video/eval/choose_best_ckpt/evalualtion_ckpt.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=val%j
+#SBATCH --output=val%j.out
+#SBATCH --error=val%j.err
+#SBATCH --time=0-10:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+## run the application:
+NAME=$2 # Name of the experiment
+DATASET="dataset_name" # available datasets: tvqa, msrvtt, msvd, activitynet,tgif,video_chatgpt_generic,video_chatgpt_temporal,video_chatgpt_consistency 
+BATCH_SIZE=2  # batch size
+CKPT_PATH=$1 # path to the checkpoint
+cfg_path="test_configs/mistral_test_config.yaml" # path to the config file
+cd ../../../
+python eval_video.py --dataset $DATASET --batch_size $BATCH_SIZE --name $NAME --ckpt $CKPT_PATH  --cfg-path=$cfg_path --add_subtitles

jobs_video/eval/llama2_evalualtion.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=llama2_best%j
+#SBATCH --output=llama2_best%j.out
+#SBATCH --error=llama2_best%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+## run the application:
+NAME="llama2_best" # Name of the experiment
+DATASET="tvqa" # available datasets: tvqa, msrvtt, msvd, activitynet,tgif ,video_chatgpt_generic,video_chatgpt_temporal,video_chatgpt_consistency
+BATCH_SIZE=8 
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth" # path to the checkpoint
+cfg_path="test_configs/llama2_test_config.yaml" # path to the config file
+# # if the number of samples are large you can specify the start and end index to evaluate on several machines
+# pass the start and end index as arguments
+start=$1 # start index
+end=$2 # end index
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=10000000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+cd ../../
+# without subtitles
+python eval_video.py --dataset $DATASET --batch_size $BATCH_SIZE --name $NAME --ckpt $CKPT_PATH  --cfg-path=$cfg_path --start $start --end $end
+
+# with subtitles
+# python eval_video.py --dataset $DATASET --batch_size $BATCH_SIZE --name $NAME --ckpt $CKPT_PATH  --cfg-path=$cfg_path --add_subtitles --start $start --end $end

jobs_video/eval/mistral_evalualtion.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --mail-user=kirolos.ataallah@kaust.edu.sa
+#SBATCH --mail-type=ALL
+#SBATCH --job-name=mistral_best%j
+#SBATCH --output=mistral_best%j.out
+#SBATCH --error=mistral_best%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+## run the application:
+NAME="mistral_best" # Name of the experiment
+DATASET="tvqa" # available datasets: tvqa, msrvtt, msvd, activitynet,tgif,video_chatgpt_generic,video_chatgpt_temporal,video_chatgpt_consistency
+BATCH_SIZE=4 # batch size for A100 by using subtiles is 2 and without subtitles is 4 
+CKPT_PATH="checkpoints/video_mistral_checkpoint_best.pth" # path to the checkpoint
+cfg_path="test_configs/mistral_test_config.yaml" # path to the config file
+# # if the number of samples are large you can specify the start and end index to evaluate on several machines
+# pass the start and end index as arguments
+start=$1 # start index
+end=$2 # end index
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=10000000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+cd ../../
+# without subtitles
+python eval_video.py --dataset $DATASET --batch_size $BATCH_SIZE --name $NAME --ckpt $CKPT_PATH  --cfg-path=$cfg_path  --start $start --end $end
+
+# with subtitles
+# python eval_video.py --dataset $DATASET --batch_size $BATCH_SIZE --name $NAME --ckpt $CKPT_PATH  --cfg-path=$cfg_path --add_subtitles --start $start --end $end

jobs_video/eval/submit_job.py

@@ -0,0 +1,19 @@
+import os 
+import shutil
+import sys
+
+start=0
+end=7800
+step=800
+
+# Mistral
+for i in range(start,end,step):
+    cmd=f'sbatch ./mistral_evalualtion.sh {i} {i+step}'
+    # print(cmd)
+    os.system(cmd)
+  
+# Llama 2  
+# for i in range(start,end,step):
+#     cmd=f'sbatch ./llama2_evalualtion.sh {i} {i+step}'
+#     # print(cmd)
+#     os.system(cmd)

jobs_video/train/stage_2_llama2.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=test
+#SBATCH --output=test.out
+#SBATCH --error=test.err
+#SBATCH --time=23:00:00
+#SBATCH --mem=110G
+#SBATCH --gres=gpu:a100:4
+#SBATCH --cpus-per-task=16
+## run the application:
+job_name=test # Name of the experiment
+cfg_path="train_configs/224_v2_llama2_video_stage_2.yaml" # path to the config file
+number_of_gpus=1 # number of gpus
+# cd ../../
+
+read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range
+while :
+do
+        PORT="`shuf -i $LOWERPORT-$UPPERPORT -n 1`"
+        ss -lpn | grep -q ":$PORT " || break
+done
+echo "Port is $PORT"
+torchrun --master-port ${PORT} --nproc-per-node $number_of_gpus train.py --job_name ${job_name} --cfg-path ${cfg_path}

jobs_video/train/stage_2_mistral.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=test
+#SBATCH --output=test.out
+#SBATCH --error=test.err
+#SBATCH --time=23:00:00
+#SBATCH --mem=110G
+#SBATCH --gres=gpu:a100:4
+#SBATCH --cpus-per-task=16
+## run the application:
+job_name=test # Name of the experiment
+cfg_path="train_configs/224_v2_mistral_video_stage_2.yaml" # path to the config file
+number_of_gpus=1 # number of gpus
+# cd ../../
+
+read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range
+while :
+do
+        PORT="`shuf -i $LOWERPORT-$UPPERPORT -n 1`"
+        ss -lpn | grep -q ":$PORT " || break
+done
+echo "Port is $PORT"
+torchrun --master-port ${PORT} --nproc-per-node $number_of_gpus train.py --job_name ${job_name} --cfg-path ${cfg_path}

jobs_video/train/stage_3_llama2.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=test
+#SBATCH --output=test.out
+#SBATCH --error=test.err
+#SBATCH --time=23:00:00
+#SBATCH --mem=110G
+#SBATCH --gres=gpu:a100:4
+#SBATCH --cpus-per-task=16
+## run the application:
+job_name="test" # Name of the experiment
+cfg_path="train_configs/224_v2_llama2_video_stage_3.yaml" # path to the config file
+number_of_gpus=1 # number of gpus
+# cd ../../
+
+read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range
+while :
+do
+        PORT="`shuf -i $LOWERPORT-$UPPERPORT -n 1`"
+        ss -lpn | grep -q ":$PORT " || break
+done
+echo "Port is $PORT"
+torchrun --master-port ${PORT} --nproc-per-node $number_of_gpus train.py --job_name ${job_name} --cfg-path ${cfg_path}

jobs_video/train/stage_3_mistral.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=test
+#SBATCH --output=test.out
+#SBATCH --error=test.err
+#SBATCH --time=23:00:00
+#SBATCH --mem=110G
+#SBATCH --gres=gpu:a100:4
+#SBATCH --cpus-per-task=16
+## run the application:
+job_name="test" # Name of the experiment
+cfg_path="train_configs/224_v2_mistral_video_stage_3.yaml" # path to the config file
+number_of_gpus=1 # number of gpus
+# cd ../../
+
+read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range
+while :
+do
+        PORT="`shuf -i $LOWERPORT-$UPPERPORT -n 1`"
+        ss -lpn | grep -q ":$PORT " || break
+done
+echo "Port is $PORT"
+torchrun --master-port ${PORT} --nproc-per-node $number_of_gpus train.py --job_name ${job_name} --cfg-path ${cfg_path}

minigpt4/__init__.py

@@ -0,0 +1,31 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import os
+import sys
+
+from omegaconf import OmegaConf
+
+from minigpt4.common.registry import registry
+
+from minigpt4.datasets.builders import *
+from minigpt4.models import *
+from minigpt4.processors import *
+from minigpt4.tasks import *
+
+
+root_dir = os.path.dirname(os.path.abspath(__file__))
+default_cfg = OmegaConf.load(os.path.join(root_dir, "configs/default.yaml"))
+
+registry.register_path("library_root", root_dir)
+repo_root = os.path.join(root_dir, "..")
+registry.register_path("repo_root", repo_root)
+cache_root = os.path.join(repo_root, default_cfg.env.cache_root)
+registry.register_path("cache_root", cache_root)
+
+registry.register("MAX_INT", sys.maxsize)
+registry.register("SPLIT_NAMES", ["train", "val", "test"])

minigpt4/common/config.py

@@ -0,0 +1,474 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import logging
+import json
+from typing import Dict
+
+from omegaconf import OmegaConf
+from minigpt4.common.registry import registry
+
+
+class Config:
+    def __init__(self, args):
+        self.config = {}
+
+        self.args = args
+
+        # Register the config and configuration for setup
+        registry.register("configuration", self)
+
+        user_config = self._build_opt_list(self.args.options)
+
+        config = OmegaConf.load(self.args.cfg_path)
+
+        runner_config = self.build_runner_config(config)
+        model_config = self.build_model_config(config, **user_config)
+        dataset_config = self.build_dataset_config(config)
+
+        # Validate the user-provided runner configuration
+        # model and dataset configuration are supposed to be validated by the respective classes
+        # [TODO] validate the model/dataset configuration
+        # self._validate_runner_config(runner_config)
+
+        # Override the default configuration with user options.
+        self.config = OmegaConf.merge(
+            runner_config, model_config, dataset_config, user_config
+        )
+
+    def _validate_runner_config(self, runner_config):
+        """
+        This method validates the configuration, such that
+            1) all the user specified options are valid;
+            2) no type mismatches between the user specified options and the config.
+        """
+        runner_config_validator = create_runner_config_validator()
+        runner_config_validator.validate(runner_config)
+
+    def _build_opt_list(self, opts):
+        opts_dot_list = self._convert_to_dot_list(opts)
+        return OmegaConf.from_dotlist(opts_dot_list)
+
+    @staticmethod
+    def build_model_config(config, **kwargs):
+        model = config.get("model", None)
+        assert model is not None, "Missing model configuration file."
+
+        model_cls = registry.get_model_class(model.arch)
+        assert model_cls is not None, f"Model '{model.arch}' has not been registered."
+
+        model_type = kwargs.get("model.model_type", None)
+        if not model_type:
+            model_type = model.get("model_type", None)
+        # else use the model type selected by user.
+
+        assert model_type is not None, "Missing model_type."
+
+        print("--------------")
+        print("model arch",model.arch)
+        print("model cls",model_cls)
+
+        model_config_path = model_cls.default_config_path(model_type=model_type)
+
+        model_config = OmegaConf.create()
+        # hierarchy override, customized config > default config
+        model_config = OmegaConf.merge(
+            model_config,
+            OmegaConf.load(model_config_path),
+            {"model": config["model"]},
+        )
+
+        return model_config
+
+    @staticmethod
+    def build_runner_config(config):
+        return {"run": config.run}
+
+    @staticmethod
+    def build_dataset_config(config):
+        datasets = config.get("datasets", None)
+        if datasets is None:
+            raise KeyError(
+                "Expecting 'datasets' as the root key for dataset configuration."
+            )
+
+        dataset_config = OmegaConf.create()
+
+        for dataset_name in datasets:
+
+            print("dataset name", dataset_name)
+            builder_cls = registry.get_builder_class(dataset_name)
+
+            dataset_config_type = datasets[dataset_name].get("type", "default")
+            dataset_config_path = builder_cls.default_config_path(
+                type=dataset_config_type
+            )
+
+            # hierarchy override, customized config > default config
+            dataset_config = OmegaConf.merge(
+                dataset_config,
+                OmegaConf.load(dataset_config_path),
+                {"datasets": {dataset_name: config["datasets"][dataset_name]}},
+            )
+
+        return dataset_config
+
+    def _convert_to_dot_list(self, opts):
+        if opts is None:
+            opts = []
+
+        if len(opts) == 0:
+            return opts
+
+        has_equal = opts[0].find("=") != -1
+
+        if has_equal:
+            return opts
+
+        return [(opt + "=" + value) for opt, value in zip(opts[0::2], opts[1::2])]
+
+    def get_config(self):
+        return self.config
+
+    @property
+    def run_cfg(self):
+        return self.config.run
+
+    @property
+    def datasets_cfg(self):
+        return self.config.datasets
+
+    @property
+    def model_cfg(self):
+        return self.config.model
+
+    def pretty_print(self):
+        logging.info("\n=====  Running Parameters    =====")
+        logging.info(self._convert_node_to_json(self.config.run))
+
+        logging.info("\n======  Dataset Attributes  ======")
+        datasets = self.config.datasets
+
+        for dataset in datasets:
+            if dataset in self.config.datasets:
+                logging.info(f"\n======== {dataset} =======")
+                dataset_config = self.config.datasets[dataset]
+                logging.info(self._convert_node_to_json(dataset_config))
+            else:
+                logging.warning(f"No dataset named '{dataset}' in config. Skipping")
+
+        logging.info(f"\n======  Model Attributes  ======")
+        logging.info(self._convert_node_to_json(self.config.model))
+
+    def _convert_node_to_json(self, node):
+        container = OmegaConf.to_container(node, resolve=True)
+        return json.dumps(container, indent=4, sort_keys=True)
+
+    def to_dict(self):
+        return OmegaConf.to_container(self.config)
+
+
+def node_to_dict(node):
+    return OmegaConf.to_container(node)
+
+
+class ConfigValidator:
+    """
+    This is a preliminary implementation to centralize and validate the configuration.
+    May be altered in the future.
+
+    A helper class to validate configurations from yaml file.
+
+    This serves the following purposes:
+        1. Ensure all the options in the yaml are defined, raise error if not.
+        2. when type mismatches are found, the validator will raise an error.
+        3. a central place to store and display helpful messages for supported configurations.
+
+    """
+
+    class _Argument:
+        def __init__(self, name, choices=None, type=None, help=None):
+            self.name = name
+            self.val = None
+            self.choices = choices
+            self.type = type
+            self.help = help
+
+        def __str__(self):
+            s = f"{self.name}={self.val}"
+            if self.type is not None:
+                s += f", ({self.type})"
+            if self.choices is not None:
+                s += f", choices: {self.choices}"
+            if self.help is not None:
+                s += f", ({self.help})"
+            return s
+
+    def __init__(self, description):
+        self.description = description
+
+        self.arguments = dict()
+
+        self.parsed_args = None
+
+    def __getitem__(self, key):
+        assert self.parsed_args is not None, "No arguments parsed yet."
+
+        return self.parsed_args[key]
+
+    def __str__(self) -> str:
+        return self.format_help()
+
+    def add_argument(self, *args, **kwargs):
+        """
+        Assume the first argument is the name of the argument.
+        """
+        self.arguments[args[0]] = self._Argument(*args, **kwargs)
+
+    def validate(self, config=None):
+        """
+        Convert yaml config (dict-like) to list, required by argparse.
+        """
+        for k, v in config.items():
+            assert (
+                k in self.arguments
+            ), f"""{k} is not a valid argument. Support arguments are {self.format_arguments()}."""
+
+            if self.arguments[k].type is not None:
+                try:
+                    self.arguments[k].val = self.arguments[k].type(v)
+                except ValueError:
+                    raise ValueError(f"{k} is not a valid {self.arguments[k].type}.")
+
+            if self.arguments[k].choices is not None:
+                assert (
+                    v in self.arguments[k].choices
+                ), f"""{k} must be one of {self.arguments[k].choices}."""
+
+        return config
+
+    def format_arguments(self):
+        return str([f"{k}" for k in sorted(self.arguments.keys())])
+
+    def format_help(self):
+        # description + key-value pair string for each argument
+        help_msg = str(self.description)
+        return help_msg + ", available arguments: " + self.format_arguments()
+
+    def print_help(self):
+        # display help message
+        print(self.format_help())
+
+
+def create_runner_config_validator():
+    validator = ConfigValidator(description="Runner configurations")
+
+    validator.add_argument(
+        "runner",
+        type=str,
+        choices=["runner_base", "runner_iter"],
+        help="""Runner to use. The "runner_base" uses epoch-based training while iter-based
+            runner runs based on iters. Default: runner_base""",
+    )
+    # add argumetns for training dataset ratios
+    validator.add_argument(
+        "train_dataset_ratios",
+        type=Dict[str, float],
+        help="""Ratios of training dataset. This is used in iteration-based runner.
+        Do not support for epoch-based runner because how to define an epoch becomes tricky.
+        Default: None""",
+    )
+    validator.add_argument(
+        "max_iters",
+        type=float,
+        help="Maximum number of iterations to run.",
+    )
+    validator.add_argument(
+        "max_epoch",
+        type=int,
+        help="Maximum number of epochs to run.",
+    )
+    # add arguments for iters_per_inner_epoch
+    validator.add_argument(
+        "iters_per_inner_epoch",
+        type=float,
+        help="Number of iterations per inner epoch. This is required when runner is runner_iter.",
+    )
+    lr_scheds_choices = registry.list_lr_schedulers()
+    validator.add_argument(
+        "lr_sched",
+        type=str,
+        choices=lr_scheds_choices,
+        help="Learning rate scheduler to use, from {}".format(lr_scheds_choices),
+    )
+    task_choices = registry.list_tasks()
+    validator.add_argument(
+        "task",
+        type=str,
+        choices=task_choices,
+        help="Task to use, from {}".format(task_choices),
+    )
+    # add arguments for init_lr
+    validator.add_argument(
+        "init_lr",
+        type=float,
+        help="Initial learning rate. This will be the learning rate after warmup and before decay.",
+    )
+    # add arguments for min_lr
+    validator.add_argument(
+        "min_lr",
+        type=float,
+        help="Minimum learning rate (after decay).",
+    )
+    # add arguments for warmup_lr
+    validator.add_argument(
+        "warmup_lr",
+        type=float,
+        help="Starting learning rate for warmup.",
+    )
+    # add arguments for learning rate decay rate
+    validator.add_argument(
+        "lr_decay_rate",
+        type=float,
+        help="Learning rate decay rate. Required if using a decaying learning rate scheduler.",
+    )
+    # add arguments for weight decay
+    validator.add_argument(
+        "weight_decay",
+        type=float,
+        help="Weight decay rate.",
+    )
+    # add arguments for training batch size
+    validator.add_argument(
+        "batch_size_train",
+        type=int,
+        help="Training batch size.",
+    )
+    # add arguments for evaluation batch size
+    validator.add_argument(
+        "batch_size_eval",
+        type=int,
+        help="Evaluation batch size, including validation and testing.",
+    )
+    # add arguments for number of workers for data loading
+    validator.add_argument(
+        "num_workers",
+        help="Number of workers for data loading.",
+    )
+    # add arguments for warm up steps
+    validator.add_argument(
+        "warmup_steps",
+        type=int,
+        help="Number of warmup steps. Required if a warmup schedule is used.",
+    )
+    # add arguments for random seed
+    validator.add_argument(
+        "seed",
+        type=int,
+        help="Random seed.",
+    )
+    # add arguments for output directory
+    validator.add_argument(
+        "output_dir",
+        type=str,
+        help="Output directory to save checkpoints and logs.",
+    )
+    # add arguments for whether only use evaluation
+    validator.add_argument(
+        "evaluate",
+        help="Whether to only evaluate the model. If true, training will not be performed.",
+    )
+    # add arguments for splits used for training, e.g. ["train", "val"]
+    validator.add_argument(
+        "train_splits",
+        type=list,
+        help="Splits to use for training.",
+    )
+    # add arguments for splits used for validation, e.g. ["val"]
+    validator.add_argument(
+        "valid_splits",
+        type=list,
+        help="Splits to use for validation. If not provided, will skip the validation.",
+    )
+    # add arguments for splits used for testing, e.g. ["test"]
+    validator.add_argument(
+        "test_splits",
+        type=list,
+        help="Splits to use for testing. If not provided, will skip the testing.",
+    )
+    # add arguments for accumulating gradient for iterations
+    validator.add_argument(
+        "accum_grad_iters",
+        type=int,
+        help="Number of iterations to accumulate gradient for.",
+    )
+
+    # ====== distributed training ======
+    validator.add_argument(
+        "device",
+        type=str,
+        choices=["cpu", "cuda"],
+        help="Device to use. Support 'cuda' or 'cpu' as for now.",
+    )
+    validator.add_argument(
+        "world_size",
+        type=int,
+        help="Number of processes participating in the job.",
+    )
+    validator.add_argument("dist_url", type=str)
+    validator.add_argument("distributed", type=bool)
+    # add arguments to opt using distributed sampler during evaluation or not
+    validator.add_argument(
+        "use_dist_eval_sampler",
+        type=bool,
+        help="Whether to use distributed sampler during evaluation or not.",
+    )
+
+    # ====== task specific ======
+    # generation task specific arguments
+    # add arguments for maximal length of text output
+    validator.add_argument(
+        "max_len",
+        type=int,
+        help="Maximal length of text output.",
+    )
+    # add arguments for minimal length of text output
+    validator.add_argument(
+        "min_len",
+        type=int,
+        help="Minimal length of text output.",
+    )
+    # add arguments number of beams
+    validator.add_argument(
+        "num_beams",
+        type=int,
+        help="Number of beams used for beam search.",
+    )
+
+    # vqa task specific arguments
+    # add arguments for number of answer candidates
+    validator.add_argument(
+        "num_ans_candidates",
+        type=int,
+        help="""For ALBEF and BLIP, these models first rank answers according to likelihood to select answer candidates.""",
+    )
+    # add arguments for inference method
+    validator.add_argument(
+        "inference_method",
+        type=str,
+        choices=["genearte", "rank"],
+        help="""Inference method to use for question answering. If rank, requires a answer list.""",
+    )
+
+    # ====== model specific ======
+    validator.add_argument(
+        "k_test",
+        type=int,
+        help="Number of top k most similar samples from ITC/VTC selection to be tested.",
+    )
+
+    return validator

minigpt4/common/dist_utils.py

@@ -0,0 +1,146 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import datetime
+import functools
+import os
+
+import torch
+import torch.distributed as dist
+import timm.models.hub as timm_hub
+
+
+def setup_for_distributed(is_master):
+    """
+    This function disables printing when not in master process
+    """
+    import builtins as __builtin__
+
+    builtin_print = __builtin__.print
+
+    def print(*args, **kwargs):
+        force = kwargs.pop("force", False)
+        if is_master or force:
+            builtin_print(*args, **kwargs)
+
+    __builtin__.print = print
+
+
+def is_dist_avail_and_initialized():
+    if not dist.is_available():
+        return False
+    if not dist.is_initialized():
+        return False
+    return True
+
+
+def get_world_size():
+    if not is_dist_avail_and_initialized():
+        return 1
+    return dist.get_world_size()
+
+
+def get_rank():
+    if not is_dist_avail_and_initialized():
+        return 0
+    return dist.get_rank()
+
+
+def is_main_process():
+    return get_rank() == 0
+
+
+def init_distributed_mode(args):
+    if args.distributed is False:
+        print("Not using distributed mode")
+        args.rank = 0
+        return
+
+    if 'LOCAL_RANK' not in os.environ:
+        os.environ['LOCAL_RANK'] = str(args.local_rank)
+
+    if "RANK" in os.environ and "WORLD_SIZE" in os.environ:
+        args.rank = int(os.environ["RANK"])
+        args.world_size = int(os.environ["WORLD_SIZE"])
+        args.gpu = int(os.environ["LOCAL_RANK"])
+    elif "SLURM_PROCID" in os.environ:
+        args.rank = int(os.environ["SLURM_PROCID"])
+        args.gpu = args.rank % torch.cuda.device_count()
+    else:
+        print("Not using distributed mode")
+        args.distributed = False
+        args.rank = 0
+        return
+
+    args.distributed = True
+
+    torch.cuda.set_device(args.gpu)
+    args.dist_backend = "nccl"
+    print(
+        "| distributed init (rank {}, world {}): {}".format(
+            args.rank, args.world_size, args.dist_url
+        ),
+        flush=True,
+    )
+    torch.distributed.init_process_group(
+        backend=args.dist_backend,
+        init_method=args.dist_url,
+        world_size=args.world_size,
+        rank=args.rank,
+        timeout=datetime.timedelta(
+            days=365
+        ),  # allow auto-downloading and de-compressing
+    )
+    torch.distributed.barrier()
+    setup_for_distributed(args.rank == 0)
+
+
+def get_dist_info():
+    if torch.__version__ < "1.0":
+        initialized = dist._initialized
+    else:
+        initialized = dist.is_initialized()
+    if initialized:
+        rank = dist.get_rank()
+        world_size = dist.get_world_size()
+    else:  # non-distributed training
+        rank = 0
+        world_size = 1
+    return rank, world_size
+
+
+def main_process(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        rank, _ = get_dist_info()
+        if rank == 0:
+            return func(*args, **kwargs)
+
+    return wrapper
+
+
+def download_cached_file(url, check_hash=True, progress=False):
+    """
+    Download a file from a URL and cache it locally. If the file already exists, it is not downloaded again.
+    If distributed, only the main process downloads the file, and the other processes wait for the file to be downloaded.
+    """
+
+    def get_cached_file_path():
+        # a hack to sync the file path across processes
+        parts = torch.hub.urlparse(url)
+        filename = os.path.basename(parts.path)
+        cached_file = os.path.join(timm_hub.get_cache_dir(), filename)
+
+        return cached_file
+
+    if is_main_process():
+        timm_hub.download_cached_file(url, check_hash, progress)
+
+    if is_dist_avail_and_initialized():
+        dist.barrier()
+
+    return get_cached_file_path()

minigpt4/common/eval_utils.py

@@ -0,0 +1,224 @@
+import argparse
+import numpy as np
+from nltk.translate.bleu_score import sentence_bleu
+import sys 
+sys.path.append('/home/ataallka/minigpt_video/minigpt_multi_img')
+from minigpt4.common.registry import registry
+from minigpt4.common.config import Config
+
+# imports modules for registration
+from minigpt4.datasets.builders import *
+from minigpt4.models import *
+from minigpt4.processors import *
+# from minigpt4.runners import *
+from minigpt4.tasks import *
+from pycocoevalcap.cider.cider import Cider
+import os
+import openai
+from tqdm import tqdm
+import json
+import ast
+import time
+
+def eval_parser():
+    parser = argparse.ArgumentParser(description="Demo")
+    parser.add_argument("--cfg-path", help="path to configuration file.",default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--ckpt", type=str,default='checkpoints/video_llama_checkpoint_last.pth', help="path to checkpoint")
+    parser.add_argument("--eval_opt", type=str, default='all', help="path to configuration file.")
+    parser.add_argument("--max_new_tokens", type=int, default=512, help="max number of generated tokens")
+    parser.add_argument("--lora_r", type=int, default=64, help="lora rank of the model")
+    parser.add_argument("--lora_alpha", type=int, default=16, help="lora alpha")
+    parser.add_argument(
+        "--options",
+        nargs="+",
+        help="override some settings in the used config, the key-value pair "
+             "in xxx=yyy format will be merged into config file (deprecate), "
+             "change to --cfg-options instead.",
+    )
+    return parser
+
+
+def prepare_texts(texts, conv_temp, template='<Img><ImageHere></Img>', lengths=None):
+    convs = [conv_temp.copy() for _ in range(len(texts))]
+    if lengths is None:
+        [conv.append_message(conv.roles[0], '{} {}'.format(template, text)) for conv, text in zip(convs, texts)]
+    else:
+        templates = [template * length for length in lengths]
+        [conv.append_message(conv.roles[0], '{} {}'.format(template, text)) for template, conv, text in zip(templates, convs, texts)]
+    [conv.append_message(conv.roles[1], None) for conv in convs]
+    texts = [conv.get_prompt() for conv in convs]
+    return texts
+
+
+def init_model(args):
+    print('Initialization Model')
+    cfg = Config(args)
+    cfg.model_cfg.ckpt = args.ckpt
+    cfg.model_cfg.lora_r = args.lora_r
+    cfg.model_cfg.lora_alpha = args.lora_alpha
+
+    model_config = cfg.model_cfg
+    model_config.low_resource = True
+    model_cls = registry.get_model_class(model_config.arch)
+    model = model_cls.from_config(model_config).to('cuda:0')
+
+#     import pudb; pudb.set_trace()
+    key = list(cfg.datasets_cfg.keys())[0]
+    vis_processor_cfg = cfg.datasets_cfg.get(key).vis_processor.train
+    print(vis_processor_cfg)
+    vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
+    print('Initialization Finished')
+    return model, vis_processor
+
+def computeIoU(bbox1, bbox2):
+    x1, y1, x2, y2 = bbox1
+    x3, y3, x4, y4 = bbox2
+    intersection_x1 = max(x1, x3)
+    intersection_y1 = max(y1, y3)
+    intersection_x2 = min(x2, x4)
+    intersection_y2 = min(y2, y4)
+    intersection_area = max(0, intersection_x2 - intersection_x1 + 1) * max(0, intersection_y2 - intersection_y1 + 1)
+    bbox1_area = (x2 - x1 + 1) * (y2 - y1 + 1)
+    bbox2_area = (x4 - x3 + 1) * (y4 - y3 + 1)
+    union_area = bbox1_area + bbox2_area - intersection_area
+    iou = intersection_area / union_area
+    return iou
+
+def eval_bleu(results):
+    bleus1,bleus2,bleus3,bleus4 = [],[],[],[]
+    for result in tqdm (results,desc="bleu_eval"):
+        gt = result['gt']
+        pred = result['pred']
+        bleus1.append(sentence_bleu([gt.split()], pred.split(), weights=(1,0,0,0)))
+        bleus2.append(sentence_bleu([gt.split()], pred.split(), weights=(0.5,0.5,0,0)))
+        bleus3.append(sentence_bleu([gt.split()], pred.split(), weights=(0.33,0.33,0.33,0)))
+        bleus4.append(sentence_bleu([gt.split()], pred.split()))
+    # print(np.mean(bleus1),np.mean(bleus2),np.mean(bleus3),np.mean(bleus4),flush=True)
+    return {'bleu1':np.mean(bleus1),'bleu2':np.mean(bleus2),'bleu3':np.mean(bleus3),'bleu4':np.mean(bleus4)}
+
+# Create a Cider object
+cider_scorer = Cider()
+def eval_cider(pred_result,gt_result):
+    # Compute CIDEr scores
+    mean_cider_scores, cider_scores = cider_scorer.compute_score(gt_result, pred_result)
+    cider_scores_dict={}
+    for score,pred_vid_id,gt_vid_id in tqdm(zip(cider_scores.tolist(),pred_result,gt_result),desc="cider_eval") :
+        assert pred_vid_id==gt_vid_id
+        cider_scores_dict[pred_vid_id] = score
+    return {'mean_cider_scores':mean_cider_scores,'cider_scores':cider_scores_dict}
+
+
+openai.api_key_path = "/home/ataallka/chatgpt_api.txt"
+
+
+def chat_gpt_eval(results,output_path):
+    trial=0
+    gpt_results=[]
+    avg_chatgpt_score=0
+    existed_files={}
+    # read previous results from output path
+    for file in os.listdir(output_path):
+        if file.endswith(".json"):
+            with open(f'{output_path}/{file}') as json_file:
+                data = json.load(json_file)
+                gpt_results.append(data[0])
+                avg_chatgpt_score+=float(data[0]['chatgpt_score'])
+                existed_files[data[0]['video_name']]=True
+    length_output_path=len(os.listdir(output_path))
+    while len (results)!= length_output_path:
+        for res in tqdm(results,desc="chatgpt_eval"):
+            if existed_files.get(res['video_name'],False):
+                continue
+            video_name=res['video_name']
+            sentence_1=res['A']
+            sentence_2=res['pred']
+            try:
+                # prompt=f"given these 2 sentences the first one is the ground truth text and the second sentence is the generated text ,give me a score from 0 to 1 to evaluate how much they are similar to each other, and have the same context and related to each other to evaluate the quality of this generated text.the output should be only the score float number without any additional information\nfirst sentence: {sentence_1}\nsecond sentence: {sentence_2}\nscore:"
+                prompt=f"given these 2 sentences the first one is the ground truth descrption of a video and the second sentence is the generated text from a video summarization model,give it a score from 0 to 5 to evaluate the model summarization performance.the output should be only the score number without any additional information\nfirst sentence: {sentence_1}\nsecond sentence: {sentence_2}\nscore:"        
+                response = openai.ChatCompletion.create(
+                    model="gpt-3.5-turbo",
+                    messages=[
+                            {
+                            "role": "user",
+                            "content": prompt
+                            }],
+                )
+                res['chatgpt_score']=response.choices[0].message['content']
+                out={'video_name':video_name,'chatgpt_score':response.choices[0].message['content']}
+                gpt_results.append(out)
+                # save each video result in a json file
+                with open(f'{output_path}/{video_name}.json', 'w') as f:
+                    json.dump([out], f)
+                avg_chatgpt_score+=float(response.choices[0].message['content'])
+            except Exception as e:
+                print("chat gpt error",e)
+        print ("Finished chat gpt evaluation in trial",trial)
+        trial+=1
+        length_output_path=len(os.listdir(output_path))
+    return results,avg_chatgpt_score/len(results)
+def GPT4_answer(question, answer,pred):
+        try:
+            # Compute the correctness score
+            completion = openai.ChatCompletion.create(
+                # model="gpt-3.5-turbo",
+                model='gpt-4',
+                messages=[
+                    {
+                        "role": "system",
+                        "content": 
+                            "You are an intelligent chatbot designed for evaluating the correctness of generative outputs for question-answer pairs. "
+                            "Your task is to compare the predicted answer with the correct answer and determine if they match meaningfully. Here's how you can accomplish the task:"
+                            "------"
+                            "##INSTRUCTIONS: "
+                            "- Focus on the meaningful match between the predicted answer and the correct answer.\n"
+                            "- Consider synonyms or paraphrases as valid matches.\n"
+                            "- Evaluate the correctness of the prediction compared to the answer."
+                    },
+                    {
+                        "role": "user",
+                        "content":
+                            "Please evaluate the following video-based question-answer pair:\n\n"
+                            f"Question: {question}\n"
+                            f"Correct Answer: {answer}\n"
+                            f"Predicted Answer: {pred}\n\n"
+                            "Provide your evaluation only as a yes/no and score where the score is an integer value between 0 and 5, with 5 indicating the highest meaningful match. "
+                            "Please generate the response in the form of a Python dictionary string with keys 'pred' and 'score', where value of 'pred' is  a string of 'yes' or 'no' and value of 'score' is in INTEGER, not STRING."
+                            "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
+                            "For example, your response should look like this: {'pred': 'yes', 'score': 4.8}."
+                    }
+                ]
+            )
+            # Convert response to a Python dictionary.
+            response_message = completion["choices"][0]["message"]["content"]
+            response_dict = ast.literal_eval(response_message)
+            return response_dict
+        except Exception as e:
+            print(f"Error : {e}")
+            return None
+def GPT4_evaluation(val_result):
+    scores=[]
+    yes_count=0
+    no_count=0
+    for res in val_result:
+        gpt_response=GPT4_answer(res['Q'],res['A'],res['pred'])
+        if gpt_response is None:
+            continue
+        try:
+            scores.append(float(gpt_response['score']))
+            if 'yes' in gpt_response['pred'].lower():
+                yes_count+=1
+            elif 'no' in gpt_response['pred'].lower():
+                no_count+=1
+        except:
+            continue
+    avg_score=sum(scores)/len(scores)
+    accuracy=(yes_count/(yes_count+no_count))*100
+    print(f"chatgpt score: {avg_score} accuracy: {accuracy}")
+    return avg_score,accuracy
+
+# with open('results/ckpt_15_res89_res32_Video_validation_Dataset_subtitles.json','r') as f:
+#     results = json.load(f)
+# t1=time.time()
+# avg_score,accuracy=GPT4_evaluation(results)
+# print(f"chatgpt score: {avg_score} accuracy: {accuracy}")
+# print(f"Time taken: {time.time()-t1}")

minigpt4/common/gradcam.py

@@ -0,0 +1,24 @@
+import numpy as np
+from matplotlib import pyplot as plt
+from scipy.ndimage import filters
+from skimage import transform as skimage_transform
+
+
+def getAttMap(img, attMap, blur=True, overlap=True):
+    attMap -= attMap.min()
+    if attMap.max() > 0:
+        attMap /= attMap.max()
+    attMap = skimage_transform.resize(attMap, (img.shape[:2]), order=3, mode="constant")
+    if blur:
+        attMap = filters.gaussian_filter(attMap, 0.02 * max(img.shape[:2]))
+        attMap -= attMap.min()
+        attMap /= attMap.max()
+    cmap = plt.get_cmap("jet")
+    attMapV = cmap(attMap)
+    attMapV = np.delete(attMapV, 3, 2)
+    if overlap:
+        attMap = (
+            1 * (1 - attMap**0.7).reshape(attMap.shape + (1,)) * img
+            + (attMap**0.7).reshape(attMap.shape + (1,)) * attMapV
+        )
+    return attMap

minigpt4/common/logger.py

@@ -0,0 +1,195 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import datetime
+import logging
+import time
+from collections import defaultdict, deque
+
+import torch
+import torch.distributed as dist
+
+from minigpt4.common import dist_utils
+
+
+class SmoothedValue(object):
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = "{median:.4f} ({global_avg:.4f})"
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, n=1):
+        self.deque.append(value)
+        self.count += n
+        self.total += value * n
+
+    def synchronize_between_processes(self):
+        """
+        Warning: does not synchronize the deque!
+        """
+        if not dist_utils.is_dist_avail_and_initialized():
+            return
+        t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda")
+        dist.barrier()
+        dist.all_reduce(t)
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median,
+            avg=self.avg,
+            global_avg=self.global_avg,
+            max=self.max,
+            value=self.value,
+        )
+
+
+class MetricLogger(object):
+    def __init__(self, delimiter="\t"):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError(
+            "'{}' object has no attribute '{}'".format(type(self).__name__, attr)
+        )
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append("{}: {}".format(name, str(meter)))
+        return self.delimiter.join(loss_str)
+
+    def global_avg(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append("{}: {:.4f}".format(name, meter.global_avg))
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ""
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt="{avg:.4f}")
+        data_time = SmoothedValue(fmt="{avg:.4f}")
+        space_fmt = ":" + str(len(str(len(iterable)))) + "d"
+        log_msg = [
+            header,
+            "[{0" + space_fmt + "}/{1}]",
+            "eta: {eta}",
+            "{meters}",
+            "time: {time}",
+            "data: {data}",
+        ]
+        if torch.cuda.is_available():
+            log_msg.append("max mem: {memory:.0f}")
+        log_msg = self.delimiter.join(log_msg)
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0 or i == len(iterable) - 1:
+                eta_seconds = iter_time.global_avg * (len(iterable) - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(
+                        log_msg.format(
+                            i,
+                            len(iterable),
+                            eta=eta_string,
+                            meters=str(self),
+                            time=str(iter_time),
+                            data=str(data_time),
+                            memory=torch.cuda.max_memory_allocated() / MB,
+                        )
+                    )
+                else:
+                    print(
+                        log_msg.format(
+                            i,
+                            len(iterable),
+                            eta=eta_string,
+                            meters=str(self),
+                            time=str(iter_time),
+                            data=str(data_time),
+                        )
+                    )
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print(
+            "{} Total time: {} ({:.4f} s / it)".format(
+                header, total_time_str, total_time / len(iterable)
+            )
+        )
+
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def setup_logger():
+    logging.basicConfig(
+        level=logging.INFO if dist_utils.is_main_process() else logging.WARN,
+        format="%(asctime)s [%(levelname)s] %(message)s",
+        handlers=[logging.StreamHandler()],
+    )

minigpt4/common/optims.py

@@ -0,0 +1,119 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import math
+
+from minigpt4.common.registry import registry
+
+
+@registry.register_lr_scheduler("linear_warmup_step_lr")
+class LinearWarmupStepLRScheduler:
+    def __init__(
+        self,
+        optimizer,
+        max_epoch,
+        min_lr,
+        init_lr,
+        decay_rate=1,
+        warmup_start_lr=-1,
+        warmup_steps=0,
+        **kwargs
+    ):
+        self.optimizer = optimizer
+
+        self.max_epoch = max_epoch
+        self.min_lr = min_lr
+
+        self.decay_rate = decay_rate
+
+        self.init_lr = init_lr
+        self.warmup_steps = warmup_steps
+        self.warmup_start_lr = warmup_start_lr if warmup_start_lr >= 0 else init_lr
+
+    def step(self, cur_epoch, cur_step):
+        if cur_epoch == 0:
+            warmup_lr_schedule(
+                step=cur_step,
+                optimizer=self.optimizer,
+                max_step=self.warmup_steps,
+                init_lr=self.warmup_start_lr,
+                max_lr=self.init_lr,
+            )
+        else:
+            step_lr_schedule(
+                epoch=cur_epoch,
+                optimizer=self.optimizer,
+                init_lr=self.init_lr,
+                min_lr=self.min_lr,
+                decay_rate=self.decay_rate,
+            )
+
+
+@registry.register_lr_scheduler("linear_warmup_cosine_lr")
+class LinearWarmupCosineLRScheduler:
+    def __init__(
+        self,
+        optimizer,
+        max_epoch,
+        iters_per_epoch,
+        min_lr,
+        init_lr,
+        warmup_steps=0,
+        warmup_start_lr=-1,
+        **kwargs
+    ):
+        self.optimizer = optimizer
+
+        self.max_epoch = max_epoch
+        self.iters_per_epoch = iters_per_epoch
+        self.min_lr = min_lr
+
+        self.init_lr = init_lr
+        self.warmup_steps = warmup_steps
+        self.warmup_start_lr = warmup_start_lr if warmup_start_lr >= 0 else init_lr
+
+    def step(self, cur_epoch, cur_step):
+        total_cur_step = cur_epoch * self.iters_per_epoch + cur_step
+        if total_cur_step < self.warmup_steps:
+            warmup_lr_schedule(
+                step=total_cur_step,
+                optimizer=self.optimizer,
+                max_step=self.warmup_steps,
+                init_lr=self.warmup_start_lr,
+                max_lr=self.init_lr,
+            )
+        else:
+            cosine_lr_schedule(
+                epoch=total_cur_step,
+                optimizer=self.optimizer,
+                max_epoch=self.max_epoch * self.iters_per_epoch,
+                init_lr=self.init_lr,
+                min_lr=self.min_lr,
+            )
+
+
+def cosine_lr_schedule(optimizer, epoch, max_epoch, init_lr, min_lr):
+    """Decay the learning rate"""
+    lr = (init_lr - min_lr) * 0.5 * (
+        1.0 + math.cos(math.pi * epoch / max_epoch)
+    ) + min_lr
+    for param_group in optimizer.param_groups:
+        param_group["lr"] = lr
+
+
+def warmup_lr_schedule(optimizer, step, max_step, init_lr, max_lr):
+    """Warmup the learning rate"""
+    lr = min(max_lr, init_lr + (max_lr - init_lr) * step / max(max_step, 1))
+    for param_group in optimizer.param_groups:
+        param_group["lr"] = lr
+
+
+def step_lr_schedule(optimizer, epoch, init_lr, min_lr, decay_rate):
+    """Decay the learning rate"""
+    lr = max(min_lr, init_lr * (decay_rate**epoch))
+    for param_group in optimizer.param_groups:
+        param_group["lr"] = lr

minigpt4/common/registry.py

@@ -0,0 +1,330 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+
+class Registry:
+    mapping = {
+        "builder_name_mapping": {},
+        "task_name_mapping": {},
+        "processor_name_mapping": {},
+        "model_name_mapping": {},
+        "lr_scheduler_name_mapping": {},
+        "runner_name_mapping": {},
+        "state": {},
+        "paths": {},
+    }
+
+    @classmethod
+    def register_builder(cls, name):
+        r"""Register a dataset builder to registry with key 'name'
+
+        Args:
+            name: Key with which the builder will be registered.
+
+        Usage:
+
+            from minigpt4.common.registry import registry
+            from minigpt4.datasets.base_dataset_builder import BaseDatasetBuilder
+        """
+
+        def wrap(builder_cls):
+            from minigpt4.datasets.builders.base_dataset_builder import BaseDatasetBuilder
+
+            assert issubclass(
+                builder_cls, BaseDatasetBuilder
+            ), "All builders must inherit BaseDatasetBuilder class, found {}".format(
+                builder_cls
+            )
+            if name in cls.mapping["builder_name_mapping"]:
+                raise KeyError(
+                    "Name '{}' already registered for {}.".format(
+                        name, cls.mapping["builder_name_mapping"][name]
+                    )
+                )
+            cls.mapping["builder_name_mapping"][name] = builder_cls
+            return builder_cls
+
+        return wrap
+
+    @classmethod
+    def register_task(cls, name):
+        r"""Register a task to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from minigpt4.common.registry import registry
+        """
+
+        def wrap(task_cls):
+            from minigpt4.tasks.base_task import BaseTask
+
+            assert issubclass(
+                task_cls, BaseTask
+            ), "All tasks must inherit BaseTask class"
+            if name in cls.mapping["task_name_mapping"]:
+                raise KeyError(
+                    "Name '{}' already registered for {}.".format(
+                        name, cls.mapping["task_name_mapping"][name]
+                    )
+                )
+            cls.mapping["task_name_mapping"][name] = task_cls
+            return task_cls
+
+        return wrap
+
+    @classmethod
+    def register_model(cls, name):
+        r"""Register a task to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from minigpt4.common.registry import registry
+        """
+
+        def wrap(model_cls):
+            # from minigpt4.models import BaseModel
+
+            # assert issubclass(
+            #     model_cls, BaseModel
+            # ), "All models must inherit BaseModel class"
+
+            if name in cls.mapping["model_name_mapping"]:
+                raise KeyError(
+                    "Name '{}' already registered for {}.".format(
+                        name, cls.mapping["model_name_mapping"][name]
+                    )
+                )
+            cls.mapping["model_name_mapping"][name] = model_cls
+            return model_cls
+
+        return wrap
+
+    @classmethod
+    def register_processor(cls, name):
+        r"""Register a processor to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from minigpt4.common.registry import registry
+        """
+
+        def wrap(processor_cls):
+            from minigpt4.processors import BaseProcessor
+
+            assert issubclass(
+                processor_cls, BaseProcessor
+            ), "All processors must inherit BaseProcessor class"
+            if name in cls.mapping["processor_name_mapping"]:
+                raise KeyError(
+                    "Name '{}' already registered for {}.".format(
+                        name, cls.mapping["processor_name_mapping"][name]
+                    )
+                )
+            cls.mapping["processor_name_mapping"][name] = processor_cls
+            return processor_cls
+
+        return wrap
+
+    @classmethod
+    def register_lr_scheduler(cls, name):
+        r"""Register a model to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from minigpt4.common.registry import registry
+        """
+
+        def wrap(lr_sched_cls):
+            if name in cls.mapping["lr_scheduler_name_mapping"]:
+                raise KeyError(
+                    "Name '{}' already registered for {}.".format(
+                        name, cls.mapping["lr_scheduler_name_mapping"][name]
+                    )
+                )
+            cls.mapping["lr_scheduler_name_mapping"][name] = lr_sched_cls
+            return lr_sched_cls
+
+        return wrap
+
+    @classmethod
+    def register_runner(cls, name):
+        r"""Register a model to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from minigpt4.common.registry import registry
+        """
+
+        def wrap(runner_cls):
+            if name in cls.mapping["runner_name_mapping"]:
+                raise KeyError(
+                    "Name '{}' already registered for {}.".format(
+                        name, cls.mapping["runner_name_mapping"][name]
+                    )
+                )
+            cls.mapping["runner_name_mapping"][name] = runner_cls
+            return runner_cls
+
+        return wrap
+
+    @classmethod
+    def register_path(cls, name, path):
+        r"""Register a path to registry with key 'name'
+
+        Args:
+            name: Key with which the path will be registered.
+
+        Usage:
+
+            from minigpt4.common.registry import registry
+        """
+        assert isinstance(path, str), "All path must be str."
+        if name in cls.mapping["paths"]:
+            raise KeyError("Name '{}' already registered.".format(name))
+        cls.mapping["paths"][name] = path
+
+    @classmethod
+    def register(cls, name, obj):
+        r"""Register an item to registry with key 'name'
+
+        Args:
+            name: Key with which the item will be registered.
+
+        Usage::
+
+            from minigpt4.common.registry import registry
+
+            registry.register("config", {})
+        """
+        path = name.split(".")
+        current = cls.mapping["state"]
+
+        for part in path[:-1]:
+            if part not in current:
+                current[part] = {}
+            current = current[part]
+
+        current[path[-1]] = obj
+
+    # @classmethod
+    # def get_trainer_class(cls, name):
+    #     return cls.mapping["trainer_name_mapping"].get(name, None)
+
+    @classmethod
+    def get_builder_class(cls, name):
+        return cls.mapping["builder_name_mapping"].get(name, None)
+
+    @classmethod
+    def get_model_class(cls, name):
+        return cls.mapping["model_name_mapping"].get(name, None)
+
+    @classmethod
+    def get_task_class(cls, name):
+        return cls.mapping["task_name_mapping"].get(name, None)
+
+    @classmethod
+    def get_processor_class(cls, name):
+        return cls.mapping["processor_name_mapping"].get(name, None)
+
+    @classmethod
+    def get_lr_scheduler_class(cls, name):
+        return cls.mapping["lr_scheduler_name_mapping"].get(name, None)
+
+    @classmethod
+    def get_runner_class(cls, name):
+        return cls.mapping["runner_name_mapping"].get(name, None)
+
+    @classmethod
+    def list_runners(cls):
+        return sorted(cls.mapping["runner_name_mapping"].keys())
+
+    @classmethod
+    def list_models(cls):
+        return sorted(cls.mapping["model_name_mapping"].keys())
+
+    @classmethod
+    def list_tasks(cls):
+        return sorted(cls.mapping["task_name_mapping"].keys())
+
+    @classmethod
+    def list_processors(cls):
+        return sorted(cls.mapping["processor_name_mapping"].keys())
+
+    @classmethod
+    def list_lr_schedulers(cls):
+        return sorted(cls.mapping["lr_scheduler_name_mapping"].keys())
+
+    @classmethod
+    def list_datasets(cls):
+        return sorted(cls.mapping["builder_name_mapping"].keys())
+
+    @classmethod
+    def get_path(cls, name):
+        return cls.mapping["paths"].get(name, None)
+
+    @classmethod
+    def get(cls, name, default=None, no_warning=False):
+        r"""Get an item from registry with key 'name'
+
+        Args:
+            name (string): Key whose value needs to be retrieved.
+            default: If passed and key is not in registry, default value will
+                     be returned with a warning. Default: None
+            no_warning (bool): If passed as True, warning when key doesn't exist
+                               will not be generated. Useful for MMF's
+                               internal operations. Default: False
+        """
+        original_name = name
+        name = name.split(".")
+        value = cls.mapping["state"]
+        for subname in name:
+            value = value.get(subname, default)
+            if value is default:
+                break
+
+        if (
+            "writer" in cls.mapping["state"]
+            and value == default
+            and no_warning is False
+        ):
+            cls.mapping["state"]["writer"].warning(
+                "Key {} is not present in registry, returning default value "
+                "of {}".format(original_name, default)
+            )
+        return value
+
+    @classmethod
+    def unregister(cls, name):
+        r"""Remove an item from registry with key 'name'
+
+        Args:
+            name: Key which needs to be removed.
+        Usage::
+
+            from mmf.common.registry import registry
+
+            config = registry.unregister("config")
+        """
+        return cls.mapping["state"].pop(name, None)
+
+
+registry = Registry()

minigpt4/common/utils.py

@@ -0,0 +1,424 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import io
+import json
+import logging
+import os
+import pickle
+import re
+import shutil
+import urllib
+import urllib.error
+import urllib.request
+from typing import Optional
+from urllib.parse import urlparse
+
+import numpy as np
+import pandas as pd
+import yaml
+from iopath.common.download import download
+from iopath.common.file_io import file_lock, g_pathmgr
+from minigpt4.common.registry import registry
+from torch.utils.model_zoo import tqdm
+from torchvision.datasets.utils import (
+    check_integrity,
+    download_file_from_google_drive,
+    extract_archive,
+)
+
+
+def now():
+    from datetime import datetime
+
+    return datetime.now().strftime("%Y%m%d%H%M")
+
+
+def is_url(url_or_filename):
+    parsed = urlparse(url_or_filename)
+    return parsed.scheme in ("http", "https")
+
+
+def get_cache_path(rel_path):
+    return os.path.expanduser(os.path.join(registry.get_path("cache_root"), rel_path))
+
+
+def get_abs_path(rel_path):
+    return os.path.join(registry.get_path("library_root"), rel_path)
+
+
+def load_json(filename):
+    with open(filename, "r") as f:
+        return json.load(f)
+
+
+# The following are adapted from torchvision and vissl
+# torchvision: https://github.com/pytorch/vision
+# vissl: https://github.com/facebookresearch/vissl/blob/main/vissl/utils/download.py
+
+
+def makedir(dir_path):
+    """
+    Create the directory if it does not exist.
+    """
+    is_success = False
+    try:
+        if not g_pathmgr.exists(dir_path):
+            g_pathmgr.mkdirs(dir_path)
+        is_success = True
+    except BaseException:
+        print(f"Error creating directory: {dir_path}")
+    return is_success
+
+
+def get_redirected_url(url: str):
+    """
+    Given a URL, returns the URL it redirects to or the
+    original URL in case of no indirection
+    """
+    import requests
+
+    with requests.Session() as session:
+        with session.get(url, stream=True, allow_redirects=True) as response:
+            if response.history:
+                return response.url
+            else:
+                return url
+
+
+def to_google_drive_download_url(view_url: str) -> str:
+    """
+    Utility function to transform a view URL of google drive
+    to a download URL for google drive
+    Example input:
+        https://drive.google.com/file/d/137RyRjvTBkBiIfeYBNZBtViDHQ6_Ewsp/view
+    Example output:
+        https://drive.google.com/uc?export=download&id=137RyRjvTBkBiIfeYBNZBtViDHQ6_Ewsp
+    """
+    splits = view_url.split("/")
+    assert splits[-1] == "view"
+    file_id = splits[-2]
+    return f"https://drive.google.com/uc?export=download&id={file_id}"
+
+
+def download_google_drive_url(url: str, output_path: str, output_file_name: str):
+    """
+    Download a file from google drive
+    Downloading an URL from google drive requires confirmation when
+    the file of the size is too big (google drive notifies that
+    anti-viral checks cannot be performed on such files)
+    """
+    import requests
+
+    with requests.Session() as session:
+
+        # First get the confirmation token and append it to the URL
+        with session.get(url, stream=True, allow_redirects=True) as response:
+            for k, v in response.cookies.items():
+                if k.startswith("download_warning"):
+                    url = url + "&confirm=" + v
+
+        # Then download the content of the file
+        with session.get(url, stream=True, verify=True) as response:
+            makedir(output_path)
+            path = os.path.join(output_path, output_file_name)
+            total_size = int(response.headers.get("Content-length", 0))
+            with open(path, "wb") as file:
+                from tqdm import tqdm
+
+                with tqdm(total=total_size) as progress_bar:
+                    for block in response.iter_content(
+                        chunk_size=io.DEFAULT_BUFFER_SIZE
+                    ):
+                        file.write(block)
+                        progress_bar.update(len(block))
+
+
+def _get_google_drive_file_id(url: str) -> Optional[str]:
+    parts = urlparse(url)
+
+    if re.match(r"(drive|docs)[.]google[.]com", parts.netloc) is None:
+        return None
+
+    match = re.match(r"/file/d/(?P<id>[^/]*)", parts.path)
+    if match is None:
+        return None
+
+    return match.group("id")
+
+
+def _urlretrieve(url: str, filename: str, chunk_size: int = 1024) -> None:
+    with open(filename, "wb") as fh:
+        with urllib.request.urlopen(
+            urllib.request.Request(url, headers={"User-Agent": "vissl"})
+        ) as response:
+            with tqdm(total=response.length) as pbar:
+                for chunk in iter(lambda: response.read(chunk_size), ""):
+                    if not chunk:
+                        break
+                    pbar.update(chunk_size)
+                    fh.write(chunk)
+
+
+def download_url(
+    url: str,
+    root: str,
+    filename: Optional[str] = None,
+    md5: Optional[str] = None,
+) -> None:
+    """Download a file from a url and place it in root.
+    Args:
+        url (str): URL to download file from
+        root (str): Directory to place downloaded file in
+        filename (str, optional): Name to save the file under.
+                                  If None, use the basename of the URL.
+        md5 (str, optional): MD5 checksum of the download. If None, do not check
+    """
+    root = os.path.expanduser(root)
+    if not filename:
+        filename = os.path.basename(url)
+    fpath = os.path.join(root, filename)
+
+    makedir(root)
+
+    # check if file is already present locally
+    if check_integrity(fpath, md5):
+        print("Using downloaded and verified file: " + fpath)
+        return
+
+    # expand redirect chain if needed
+    url = get_redirected_url(url)
+
+    # check if file is located on Google Drive
+    file_id = _get_google_drive_file_id(url)
+    if file_id is not None:
+        return download_file_from_google_drive(file_id, root, filename, md5)
+
+    # download the file
+    try:
+        print("Downloading " + url + " to " + fpath)
+        _urlretrieve(url, fpath)
+    except (urllib.error.URLError, IOError) as e:  # type: ignore[attr-defined]
+        if url[:5] == "https":
+            url = url.replace("https:", "http:")
+            print(
+                "Failed download. Trying https -> http instead."
+                " Downloading " + url + " to " + fpath
+            )
+            _urlretrieve(url, fpath)
+        else:
+            raise e
+
+    # check integrity of downloaded file
+    if not check_integrity(fpath, md5):
+        raise RuntimeError("File not found or corrupted.")
+
+
+def download_and_extract_archive(
+    url: str,
+    download_root: str,
+    extract_root: Optional[str] = None,
+    filename: Optional[str] = None,
+    md5: Optional[str] = None,
+    remove_finished: bool = False,
+) -> None:
+    download_root = os.path.expanduser(download_root)
+    if extract_root is None:
+        extract_root = download_root
+    if not filename:
+        filename = os.path.basename(url)
+
+    download_url(url, download_root, filename, md5)
+
+    archive = os.path.join(download_root, filename)
+    print("Extracting {} to {}".format(archive, extract_root))
+    extract_archive(archive, extract_root, remove_finished)
+
+
+def cache_url(url: str, cache_dir: str) -> str:
+    """
+    This implementation downloads the remote resource and caches it locally.
+    The resource will only be downloaded if not previously requested.
+    """
+    parsed_url = urlparse(url)
+    dirname = os.path.join(cache_dir, os.path.dirname(parsed_url.path.lstrip("/")))
+    makedir(dirname)
+    filename = url.split("/")[-1]
+    cached = os.path.join(dirname, filename)
+    with file_lock(cached):
+        if not os.path.isfile(cached):
+            logging.info(f"Downloading {url} to {cached} ...")
+            cached = download(url, dirname, filename=filename)
+    logging.info(f"URL {url} cached in {cached}")
+    return cached
+
+
+# TODO (prigoyal): convert this into RAII-style API
+def create_file_symlink(file1, file2):
+    """
+    Simply create the symlinks for a given file1 to file2.
+    Useful during model checkpointing to symlinks to the
+    latest successful checkpoint.
+    """
+    try:
+        if g_pathmgr.exists(file2):
+            g_pathmgr.rm(file2)
+        g_pathmgr.symlink(file1, file2)
+    except Exception as e:
+        logging.info(f"Could NOT create symlink. Error: {e}")
+
+
+def save_file(data, filename, append_to_json=True, verbose=True):
+    """
+    Common i/o utility to handle saving data to various file formats.
+    Supported:
+        .pkl, .pickle, .npy, .json
+    Specifically for .json, users have the option to either append (default)
+    or rewrite by passing in Boolean value to append_to_json.
+    """
+    if verbose:
+        logging.info(f"Saving data to file: {filename}")
+    file_ext = os.path.splitext(filename)[1]
+    if file_ext in [".pkl", ".pickle"]:
+        with g_pathmgr.open(filename, "wb") as fopen:
+            pickle.dump(data, fopen, pickle.HIGHEST_PROTOCOL)
+    elif file_ext == ".npy":
+        with g_pathmgr.open(filename, "wb") as fopen:
+            np.save(fopen, data)
+    elif file_ext == ".json":
+        if append_to_json:
+            with g_pathmgr.open(filename, "a") as fopen:
+                fopen.write(json.dumps(data, sort_keys=True) + "\n")
+                fopen.flush()
+        else:
+            with g_pathmgr.open(filename, "w") as fopen:
+                fopen.write(json.dumps(data, sort_keys=True) + "\n")
+                fopen.flush()
+    elif file_ext == ".yaml":
+        with g_pathmgr.open(filename, "w") as fopen:
+            dump = yaml.dump(data)
+            fopen.write(dump)
+            fopen.flush()
+    else:
+        raise Exception(f"Saving {file_ext} is not supported yet")
+
+    if verbose:
+        logging.info(f"Saved data to file: {filename}")
+
+
+def load_file(filename, mmap_mode=None, verbose=True, allow_pickle=False):
+    """
+    Common i/o utility to handle loading data from various file formats.
+    Supported:
+        .pkl, .pickle, .npy, .json
+    For the npy files, we support reading the files in mmap_mode.
+    If the mmap_mode of reading is not successful, we load data without the
+    mmap_mode.
+    """
+    if verbose:
+        logging.info(f"Loading data from file: {filename}")
+
+    file_ext = os.path.splitext(filename)[1]
+    if file_ext == ".txt":
+        with g_pathmgr.open(filename, "r") as fopen:
+            data = fopen.readlines()
+    elif file_ext in [".pkl", ".pickle"]:
+        with g_pathmgr.open(filename, "rb") as fopen:
+            data = pickle.load(fopen, encoding="latin1")
+    elif file_ext == ".npy":
+        if mmap_mode:
+            try:
+                with g_pathmgr.open(filename, "rb") as fopen:
+                    data = np.load(
+                        fopen,
+                        allow_pickle=allow_pickle,
+                        encoding="latin1",
+                        mmap_mode=mmap_mode,
+                    )
+            except ValueError as e:
+                logging.info(
+                    f"Could not mmap {filename}: {e}. Trying without g_pathmgr"
+                )
+                data = np.load(
+                    filename,
+                    allow_pickle=allow_pickle,
+                    encoding="latin1",
+                    mmap_mode=mmap_mode,
+                )
+                logging.info("Successfully loaded without g_pathmgr")
+            except Exception:
+                logging.info("Could not mmap without g_pathmgr. Trying without mmap")
+                with g_pathmgr.open(filename, "rb") as fopen:
+                    data = np.load(fopen, allow_pickle=allow_pickle, encoding="latin1")
+        else:
+            with g_pathmgr.open(filename, "rb") as fopen:
+                data = np.load(fopen, allow_pickle=allow_pickle, encoding="latin1")
+    elif file_ext == ".json":
+        with g_pathmgr.open(filename, "r") as fopen:
+            data = json.load(fopen)
+    elif file_ext == ".yaml":
+        with g_pathmgr.open(filename, "r") as fopen:
+            data = yaml.load(fopen, Loader=yaml.FullLoader)
+    elif file_ext == ".csv":
+        with g_pathmgr.open(filename, "r") as fopen:
+            data = pd.read_csv(fopen)
+    else:
+        raise Exception(f"Reading from {file_ext} is not supported yet")
+    return data
+
+
+def abspath(resource_path: str):
+    """
+    Make a path absolute, but take into account prefixes like
+    "http://" or "manifold://"
+    """
+    regex = re.compile(r"^\w+://")
+    if regex.match(resource_path) is None:
+        return os.path.abspath(resource_path)
+    else:
+        return resource_path
+
+
+def makedir(dir_path):
+    """
+    Create the directory if it does not exist.
+    """
+    is_success = False
+    try:
+        if not g_pathmgr.exists(dir_path):
+            g_pathmgr.mkdirs(dir_path)
+        is_success = True
+    except BaseException:
+        logging.info(f"Error creating directory: {dir_path}")
+    return is_success
+
+
+def is_url(input_url):
+    """
+    Check if an input string is a url. look for http(s):// and ignoring the case
+    """
+    is_url = re.match(r"^(?:http)s?://", input_url, re.IGNORECASE) is not None
+    return is_url
+
+
+def cleanup_dir(dir):
+    """
+    Utility for deleting a directory. Useful for cleaning the storage space
+    that contains various training artifacts like checkpoints, data etc.
+    """
+    if os.path.exists(dir):
+        logging.info(f"Deleting directory: {dir}")
+        shutil.rmtree(dir)
+    logging.info(f"Deleted contents of directory: {dir}")
+
+
+def get_file_size(filename):
+    """
+    Given a file, get the size of file in MB
+    """
+    size_in_mb = os.path.getsize(filename) / float(1024**2)
+    return size_in_mb

minigpt4/common/vqa_tools/VQA/PythonEvaluationTools/vqaEvalDemo.py

@@ -0,0 +1,89 @@
+# coding: utf-8
+
+import sys
+dataDir = '../../VQA'
+sys.path.insert(0, '%s/PythonHelperTools/vqaTools' %(dataDir))
+from vqa import VQA
+from vqaEvaluation.vqaEval import VQAEval
+import matplotlib.pyplot as plt
+import skimage.io as io
+import json
+import random
+import os
+
+# set up file names and paths
+versionType ='v2_' # this should be '' when using VQA v2.0 dataset
+taskType    ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0
+dataType    ='mscoco'  # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. 
+dataSubType ='train2014'
+annFile     ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType)
+quesFile    ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType)
+imgDir      ='%s/Images/%s/%s/' %(dataDir, dataType, dataSubType)
+resultType  ='fake'
+fileTypes   = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] 
+
+# An example result json file has been provided in './Results' folder.  
+
+[resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = ['%s/Results/%s%s_%s_%s_%s_%s.json'%(dataDir, versionType, taskType, dataType, dataSubType, \
+resultType, fileType) for fileType in fileTypes]  
+
+# create vqa object and vqaRes object
+vqa = VQA(annFile, quesFile)
+vqaRes = vqa.loadRes(resFile, quesFile)
+
+# create vqaEval object by taking vqa and vqaRes
+vqaEval = VQAEval(vqa, vqaRes, n=2)   #n is precision of accuracy (number of places after decimal), default is 2
+
+# evaluate results
+"""
+If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function
+By default it uses all the question ids in annotation file
+"""
+vqaEval.evaluate() 
+
+# print accuracies
+print "\n"
+print "Overall Accuracy is: %.02f\n" %(vqaEval.accuracy['overall'])
+print "Per Question Type Accuracy is the following:"
+for quesType in vqaEval.accuracy['perQuestionType']:
+	print "%s : %.02f" %(quesType, vqaEval.accuracy['perQuestionType'][quesType])
+print "\n"
+print "Per Answer Type Accuracy is the following:"
+for ansType in vqaEval.accuracy['perAnswerType']:
+	print "%s : %.02f" %(ansType, vqaEval.accuracy['perAnswerType'][ansType])
+print "\n"
+# demo how to use evalQA to retrieve low score result
+evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId]<35]   #35 is per question percentage accuracy
+if len(evals) > 0:
+	print 'ground truth answers'
+	randomEval = random.choice(evals)
+	randomAnn = vqa.loadQA(randomEval)
+	vqa.showQA(randomAnn)
+
+	print '\n'
+	print 'generated answer (accuracy %.02f)'%(vqaEval.evalQA[randomEval])
+	ann = vqaRes.loadQA(randomEval)[0]
+	print "Answer:   %s\n" %(ann['answer'])
+
+	imgId = randomAnn[0]['image_id']
+	imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'
+	if os.path.isfile(imgDir + imgFilename):
+		I = io.imread(imgDir + imgFilename)
+		plt.imshow(I)
+		plt.axis('off')
+		plt.show()
+
+# plot accuracy for various question types
+plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].values(), align='center')
+plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].keys(), rotation='0',fontsize=10)
+plt.title('Per Question Type Accuracy', fontsize=10)
+plt.xlabel('Question Types', fontsize=10)
+plt.ylabel('Accuracy', fontsize=10)
+plt.show()
+
+# save evaluation results to ./Results folder
+json.dump(vqaEval.accuracy,     open(accuracyFile,     'w'))
+json.dump(vqaEval.evalQA,       open(evalQAFile,       'w'))
+json.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w'))
+json.dump(vqaEval.evalAnsType,  open(evalAnsTypeFile,  'w'))
+

minigpt4/common/vqa_tools/VQA/PythonEvaluationTools/vqaEvaluation/__init__.py

@@ -0,0 +1 @@
+author='aagrawal'

minigpt4/common/vqa_tools/VQA/PythonEvaluationTools/vqaEvaluation/vqaEval.py

@@ -0,0 +1,192 @@
+# coding=utf-8
+
+__author__='aagrawal'
+
+import re
+# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link:
+# (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py).
+import sys
+
+
+class VQAEval:
+	def __init__(self, vqa, vqaRes, n=2):
+		self.n 			  = n
+		self.accuracy     = {}
+		self.evalQA       = {}
+		self.evalQuesType = {}
+		self.evalAnsType  = {}
+		self.vqa 		  = vqa
+		self.vqaRes       = vqaRes
+		self.params		  = {'question_id': vqa.getQuesIds()}
+		self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", \
+							 "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", \
+							 "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", \
+							 "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", \
+							 "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", \
+							 "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", \
+							 "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", \
+							 "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", \
+							 "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", \
+							 "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", \
+							 "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", \
+							 "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", \
+							 "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", \
+							 "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", \
+							 "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", \
+							 "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", \
+							 "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", \
+							 "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", \
+							 "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", \
+							 "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", \
+							 "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", \
+							 "youll": "you'll", "youre": "you're", "youve": "you've"}
+		self.manualMap    = { 'none': '0',
+							  'zero': '0',
+							  'one': '1',
+							  'two': '2',
+							  'three': '3',
+							  'four': '4',
+							  'five': '5',
+							  'six': '6',
+							  'seven': '7',
+							  'eight': '8',
+							  'nine': '9',
+							  'ten': '10'
+							}
+		self.articles     = ['a',
+							 'an',
+							 'the'
+							]
+
+
+		self.periodStrip  = re.compile("(?!<=\d)(\.)(?!\d)")
+		self.commaStrip   = re.compile("(\d)(\,)(\d)")
+		self.punct        = [';', r"/", '[', ']', '"', '{', '}',
+							 '(', ')', '=', '+', '\\', '_', '-',
+							 '>', '<', '@', '`', ',', '?', '!']
+
+
+	def evaluate(self, quesIds=None):
+		if quesIds == None:
+			quesIds = [quesId for quesId in self.params['question_id']]
+		gts = {}
+		res = {}
+		for quesId in quesIds:
+			gts[quesId] = self.vqa.qa[quesId]
+			res[quesId] = self.vqaRes.qa[quesId]
+
+		# =================================================
+		# Compute accuracy
+		# =================================================
+		accQA       = []
+		accQuesType = {}
+		accAnsType  = {}
+		# print "computing accuracy"
+		step = 0
+		for quesId in quesIds:
+			for ansDic in gts[quesId]['answers']:
+				ansDic['answer'] = ansDic['answer'].replace('\n', ' ')
+				ansDic['answer'] = ansDic['answer'].replace('\t', ' ')
+				ansDic['answer'] = ansDic['answer'].strip()
+			resAns = res[quesId]['answer']
+			resAns = resAns.replace('\n', ' ')
+			resAns = resAns.replace('\t', ' ')
+			resAns = resAns.strip()
+			gtAcc = []
+			gtAnswers = [ans['answer'] for ans in gts[quesId]['answers']]
+
+			if len(set(gtAnswers)) > 1:
+				for ansDic in gts[quesId]['answers']:
+					ansDic['answer'] = self.processPunctuation(ansDic['answer'])
+					ansDic['answer'] = self.processDigitArticle(ansDic['answer'])
+				resAns = self.processPunctuation(resAns)
+				resAns = self.processDigitArticle(resAns)
+
+			for gtAnsDatum in gts[quesId]['answers']:
+				otherGTAns = [item for item in gts[quesId]['answers'] if item!=gtAnsDatum]
+				matchingAns = [item for item in otherGTAns if item['answer'].lower()==resAns.lower()]
+				acc = min(1, float(len(matchingAns))/3)
+				gtAcc.append(acc)
+			quesType    = gts[quesId]['question_type']
+			ansType     = gts[quesId]['answer_type']
+			avgGTAcc = float(sum(gtAcc))/len(gtAcc)
+			accQA.append(avgGTAcc)
+			if quesType not in accQuesType:
+				accQuesType[quesType] = []
+			accQuesType[quesType].append(avgGTAcc)
+			if ansType not in accAnsType:
+				accAnsType[ansType] = []
+			accAnsType[ansType].append(avgGTAcc)
+			self.setEvalQA(quesId, avgGTAcc)
+			self.setEvalQuesType(quesId, quesType, avgGTAcc)
+			self.setEvalAnsType(quesId, ansType, avgGTAcc)
+			if step%100 == 0:
+				self.updateProgress(step/float(len(quesIds)))
+			step = step + 1
+
+		self.setAccuracy(accQA, accQuesType, accAnsType)
+		# print "Done computing accuracy"
+
+	def processPunctuation(self, inText):
+		outText = inText
+		for p in self.punct:
+			if (p + ' ' in inText or ' ' + p in inText) or (re.search(self.commaStrip, inText) != None):
+				outText = outText.replace(p, '')
+			else:
+				outText = outText.replace(p, ' ')
+		outText = self.periodStrip.sub("",
+									  outText,
+									  re.UNICODE)
+		return outText
+
+	def processDigitArticle(self, inText):
+		outText = []
+		tempText = inText.lower().split()
+		for word in tempText:
+			word = self.manualMap.setdefault(word, word)
+			if word not in self.articles:
+				outText.append(word)
+			else:
+				pass
+		for wordId, word in enumerate(outText):
+			if word in self.contractions:
+				outText[wordId] = self.contractions[word]
+		outText = ' '.join(outText)
+		return outText
+
+	def setAccuracy(self, accQA, accQuesType, accAnsType):
+		self.accuracy['overall']         = round(100*float(sum(accQA))/len(accQA), self.n)
+		self.accuracy['perQuestionType'] = {quesType: round(100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType}
+		self.accuracy['perAnswerType']   = {ansType:  round(100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType}
+
+	def setEvalQA(self, quesId, acc):
+		self.evalQA[quesId] = round(100*acc, self.n)
+
+	def setEvalQuesType(self, quesId, quesType, acc):
+		if quesType not in self.evalQuesType:
+			self.evalQuesType[quesType] = {}
+		self.evalQuesType[quesType][quesId] = round(100*acc, self.n)
+
+	def setEvalAnsType(self, quesId, ansType, acc):
+		if ansType not in self.evalAnsType:
+			self.evalAnsType[ansType] = {}
+		self.evalAnsType[ansType][quesId] = round(100*acc, self.n)
+
+	def updateProgress(self, progress):
+		barLength = 20
+		status = ""
+		if isinstance(progress, int):
+			progress = float(progress)
+		if not isinstance(progress, float):
+			progress = 0
+			status = "error: progress var must be float\r\n"
+		if progress < 0:
+			progress = 0
+			status = "Halt...\r\n"
+		if progress >= 1:
+			progress = 1
+			status = "Done...\r\n"
+		block = int(round(barLength*progress))
+		text = "\rFinshed Percent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), int(progress*100), status)
+		sys.stdout.write(text)
+		sys.stdout.flush()

minigpt4/common/vqa_tools/VQA/PythonHelperTools/vqaDemo.py

@@ -0,0 +1,73 @@
+# coding: utf-8
+
+from vqaTools.vqa import VQA
+import random
+import skimage.io as io
+import matplotlib.pyplot as plt
+import os
+
+dataDir		='../../VQA'
+versionType ='v2_' # this should be '' when using VQA v2.0 dataset
+taskType    ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0
+dataType    ='mscoco'  # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0.
+dataSubType ='train2014'
+annFile     ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType)
+quesFile    ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType)
+imgDir 		= '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType)
+
+# initialize VQA api for QA annotations
+vqa=VQA(annFile, quesFile)
+
+# load and display QA annotations for given question types
+"""
+All possible quesTypes for abstract and mscoco has been provided in respective text files in ../QuestionTypes/ folder.
+"""
+annIds = vqa.getQuesIds(quesTypes='how many');   
+anns = vqa.loadQA(annIds)
+randomAnn = random.choice(anns)
+vqa.showQA([randomAnn])
+imgId = randomAnn['image_id']
+imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'
+if os.path.isfile(imgDir + imgFilename):
+	I = io.imread(imgDir + imgFilename)
+	plt.imshow(I)
+	plt.axis('off')
+	plt.show()
+
+# load and display QA annotations for given answer types
+"""
+ansTypes can be one of the following
+yes/no
+number
+other
+"""
+annIds = vqa.getQuesIds(ansTypes='yes/no');   
+anns = vqa.loadQA(annIds)
+randomAnn = random.choice(anns)
+vqa.showQA([randomAnn])
+imgId = randomAnn['image_id']
+imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'
+if os.path.isfile(imgDir + imgFilename):
+	I = io.imread(imgDir + imgFilename)
+	plt.imshow(I)
+	plt.axis('off')
+	plt.show()
+
+# load and display QA annotations for given images
+"""
+Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[])
+Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types.
+"""
+ids = vqa.getImgIds()
+annIds = vqa.getQuesIds(imgIds=random.sample(ids,5));  
+anns = vqa.loadQA(annIds)
+randomAnn = random.choice(anns)
+vqa.showQA([randomAnn])  
+imgId = randomAnn['image_id']
+imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'
+if os.path.isfile(imgDir + imgFilename):
+	I = io.imread(imgDir + imgFilename)
+	plt.imshow(I)
+	plt.axis('off')
+	plt.show()
+

minigpt4/common/vqa_tools/VQA/PythonHelperTools/vqaTools/__init__.py

@@ -0,0 +1 @@
+__author__ = 'aagrawal'

minigpt4/common/vqa_tools/VQA/PythonHelperTools/vqaTools/vqa.py

@@ -0,0 +1,179 @@
+__author__ = 'aagrawal'
+__version__ = '0.9'
+
+# Interface for accessing the VQA dataset.
+
+# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: 
+# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py).
+
+# The following functions are defined:
+#  VQA        - VQA class that loads VQA annotation file and prepares data structures.
+#  getQuesIds - Get question ids that satisfy given filter conditions.
+#  getImgIds  - Get image ids that satisfy given filter conditions.
+#  loadQA     - Load questions and answers with the specified question ids.
+#  showQA     - Display the specified questions and answers.
+#  loadRes    - Load result file and create result object.
+
+# Help on each function can be accessed by: "help(COCO.function)"
+
+import json
+import datetime
+import copy
+
+
+class VQA:
+    def __init__(self, annotation_file=None, question_file=None):
+        """
+           Constructor of VQA helper class for reading and visualizing questions and answers.
+        :param annotation_file (str): location of VQA annotation file
+        :return:
+        """
+        # load dataset
+        self.dataset = {}
+        self.questions = {}
+        self.qa = {}
+        self.qqa = {}
+        self.imgToQA = {}
+        if not annotation_file == None and not question_file == None:
+            # print 'loading VQA annotations and questions into memory...'
+            time_t = datetime.datetime.utcnow()
+            dataset = json.load(open(annotation_file, 'r'))
+            questions = json.load(open(question_file, 'r'))
+            # print datetime.datetime.utcnow() - time_t
+            self.dataset = dataset
+            self.questions = questions
+            self.createIndex()
+
+    def createIndex(self):
+        imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']}
+        qa = {ann['question_id']: [] for ann in self.dataset['annotations']}
+        qqa = {ann['question_id']: [] for ann in self.dataset['annotations']}
+        for ann in self.dataset['annotations']:
+            imgToQA[ann['image_id']] += [ann]
+            qa[ann['question_id']] = ann
+        for ques in self.questions['questions']:
+            qqa[ques['question_id']] = ques
+        # print 'index created!'
+
+        # create class members
+        self.qa = qa
+        self.qqa = qqa
+        self.imgToQA = imgToQA
+
+    def info(self):
+        """
+        Print information about the VQA annotation file.
+        :return:
+        """
+
+    # for key, value in self.datset['info'].items():
+    # 	print '%s: %s'%(key, value)
+
+    def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]):
+        """
+        Get question ids that satisfy given filter conditions. default skips that filter
+        :param 	imgIds    (int array)   : get question ids for given imgs
+                quesTypes (str array)   : get question ids for given question types
+                ansTypes  (str array)   : get question ids for given answer types
+        :return:    ids   (int array)   : integer array of question ids
+        """
+        imgIds = imgIds if type(imgIds) == list else [imgIds]
+        quesTypes = quesTypes if type(quesTypes) == list else [quesTypes]
+        ansTypes = ansTypes if type(ansTypes) == list else [ansTypes]
+
+        if len(imgIds) == len(quesTypes) == len(ansTypes) == 0:
+            anns = self.dataset['annotations']
+        else:
+            if not len(imgIds) == 0:
+                anns = sum([self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA], [])
+            else:
+                anns = self.dataset['annotations']
+            anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes]
+            anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes]
+        ids = [ann['question_id'] for ann in anns]
+        return ids
+
+    def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]):
+        """
+        Get image ids that satisfy given filter conditions. default skips that filter
+        :param quesIds   (int array)   : get image ids for given question ids
+               quesTypes (str array)   : get image ids for given question types
+               ansTypes  (str array)   : get image ids for given answer types
+        :return: ids     (int array)   : integer array of image ids
+        """
+        quesIds = quesIds if type(quesIds) == list else [quesIds]
+        quesTypes = quesTypes if type(quesTypes) == list else [quesTypes]
+        ansTypes = ansTypes if type(ansTypes) == list else [ansTypes]
+
+        if len(quesIds) == len(quesTypes) == len(ansTypes) == 0:
+            anns = self.dataset['annotations']
+        else:
+            if not len(quesIds) == 0:
+                anns = sum([self.qa[quesId] for quesId in quesIds if quesId in self.qa], [])
+            else:
+                anns = self.dataset['annotations']
+            anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes]
+            anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes]
+        ids = [ann['image_id'] for ann in anns]
+        return ids
+
+    def loadQA(self, ids=[]):
+        """
+        Load questions and answers with the specified question ids.
+        :param ids (int array)       : integer ids specifying question ids
+        :return: qa (object array)   : loaded qa objects
+        """
+        if type(ids) == list:
+            return [self.qa[id] for id in ids]
+        elif type(ids) == int:
+            return [self.qa[ids]]
+
+    def showQA(self, anns):
+        """
+        Display the specified annotations.
+        :param anns (array of object): annotations to display
+        :return: None
+        """
+        if len(anns) == 0:
+            return 0
+        for ann in anns:
+            quesId = ann['question_id']
+            print("Question: %s" % (self.qqa[quesId]['question']))
+            for ans in ann['answers']:
+                print("Answer %d: %s" % (ans['answer_id'], ans['answer']))
+
+    def loadRes(self, resFile, quesFile):
+        """
+        Load result file and return a result object.
+        :param   resFile (str)     : file name of result file
+        :return: res (obj)         : result api object
+        """
+        res = VQA()
+        res.questions = json.load(open(quesFile))
+        res.dataset['info'] = copy.deepcopy(self.questions['info'])
+        res.dataset['task_type'] = copy.deepcopy(self.questions['task_type'])
+        res.dataset['data_type'] = copy.deepcopy(self.questions['data_type'])
+        res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype'])
+        res.dataset['license'] = copy.deepcopy(self.questions['license'])
+
+        # print 'Loading and preparing results...     '
+        time_t = datetime.datetime.utcnow()
+        anns = json.load(open(resFile))
+        assert type(anns) == list, 'results is not an array of objects'
+        annsQuesIds = [ann['question_id'] for ann in anns]
+        assert set(annsQuesIds) == set(self.getQuesIds()), \
+            'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.'
+        for ann in anns:
+            quesId = ann['question_id']
+            if res.dataset['task_type'] == 'Multiple Choice':
+                assert ann['answer'] in self.qqa[quesId][
+                    'multiple_choices'], 'predicted answer is not one of the multiple choices'
+            qaAnn = self.qa[quesId]
+            ann['image_id'] = qaAnn['image_id']
+            ann['question_type'] = qaAnn['question_type']
+            ann['answer_type'] = qaAnn['answer_type']
+        # print 'DONE (t=%0.2fs)'%((datetime.datetime.utcnow() - time_t).total_seconds())
+
+        res.dataset['annotations'] = anns
+        res.createIndex()
+        return res

minigpt4/common/vqa_tools/VQA/README.md

@@ -0,0 +1,80 @@
+Python API and Evaluation Code for v2.0 and v1.0 releases of the VQA dataset.
+===================
+## VQA v2.0 release ##
+This release consists of
+- Real 
+	- 82,783 MS COCO training images, 40,504 MS COCO validation images and 81,434 MS COCO testing images (images are obtained from [MS COCO website] (http://mscoco.org/dataset/#download))
+	- 443,757 questions for training, 214,354 questions for validation and 447,793 questions for testing
+	- 4,437,570 answers for training and 2,143,540 answers for validation (10 per question)
+
+There is only one type of task
+- Open-ended task
+
+## VQA v1.0 release ##
+This release consists of
+- Real 
+	- 82,783 MS COCO training images, 40,504 MS COCO validation images and 81,434 MS COCO testing images (images are obtained from [MS COCO website] (http://mscoco.org/dataset/#download))
+	- 248,349 questions for training, 121,512 questions for validation and 244,302 questions for testing (3 per image)
+	- 2,483,490 answers for training and 1,215,120 answers for validation (10 per question)
+- Abstract
+	- 20,000 training images, 10,000 validation images and 20,000 MS COCO testing images
+	- 60,000 questions for training, 30,000 questions for validation and 60,000 questions for testing (3 per image)
+	- 600,000 answers for training and 300,000 answers for validation (10 per question)
+
+There are two types of tasks
+- Open-ended task
+- Multiple-choice task (18 choices per question)
+
+## Requirements ##
+- python 2.7
+- scikit-image (visit [this page](http://scikit-image.org/docs/dev/install.html) for installation)
+- matplotlib (visit [this page](http://matplotlib.org/users/installing.html) for installation)
+
+## Files ##
+./Questions
+- For v2.0, download the question files from the [VQA download page](http://www.visualqa.org/download.html), extract them and place in this folder.
+- For v1.0, both real and abstract, question files can be found on the [VQA v1 download page](http://www.visualqa.org/vqa_v1_download.html).
+- Question files from Beta v0.9 release (123,287 MSCOCO train and val images, 369,861 questions, 3,698,610 answers) can be found below
+	- [training question files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Questions_Train_mscoco.zip)
+	- [validation question files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Questions_Val_mscoco.zip)
+- Question files from Beta v0.1 release (10k MSCOCO images, 30k questions, 300k answers) can be found [here](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.1/Questions_Train_mscoco.zip).
+
+./Annotations
+- For v2.0, download the annotations files from the [VQA download page](http://www.visualqa.org/download.html), extract them and place in this folder.
+- For v1.0, for both real and abstract, annotation files can be found on the [VQA v1 download page](http://www.visualqa.org/vqa_v1_download.html).
+- Annotation files from Beta v0.9 release (123,287 MSCOCO train and val images, 369,861 questions, 3,698,610 answers) can be found below
+	- [training annotation files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Annotations_Train_mscoco.zip)
+	- [validation annotation files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Annotations_Val_mscoco.zip)
+- Annotation files from Beta v0.1 release (10k MSCOCO images, 30k questions, 300k answers) can be found [here](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.1/Annotations_Train_mscoco.zip).
+
+./Images
+- For real, create a directory with name mscoco inside this directory. For each of train, val and test, create directories with names train2014, val2014 and test2015 respectively inside mscoco directory, download respective images from [MS COCO website](http://mscoco.org/dataset/#download) and place them in respective folders.
+- For abstract, create a directory with name abstract_v002 inside this directory. For each of train, val and test, create directories with names train2015, val2015 and test2015 respectively inside abstract_v002 directory, download respective images from [VQA download page](http://www.visualqa.org/download.html) and place them in respective folders.
+
+./PythonHelperTools
+- This directory contains the Python API to read and visualize the VQA dataset
+- vqaDemo.py (demo script)
+- vqaTools (API to read and visualize data)
+
+./PythonEvaluationTools
+- This directory contains the Python evaluation code
+- vqaEvalDemo.py (evaluation demo script)
+- vqaEvaluation (evaluation code)
+
+./Results
+- OpenEnded_mscoco_train2014_fake_results.json (an example of a fake results file for v1.0 to run the demo)
+- Visit [VQA evaluation page] (http://visualqa.org/evaluation) for more details.
+
+./QuestionTypes
+- This directory contains the following lists of question types for both real and abstract questions (question types are unchanged from v1.0 to v2.0). In a list, if there are question types of length n+k and length n with the same first n words, then the question type of length n does not include questions that belong to the question type of length n+k.
+- mscoco_question_types.txt
+- abstract_v002_question_types.txt
+
+## References ##
+- [VQA: Visual Question Answering](http://visualqa.org/)
+- [Microsoft COCO](http://mscoco.org/)
+
+## Developers ##
+- Aishwarya Agrawal (Virginia Tech)
+- Code for API is based on [MSCOCO API code](https://github.com/pdollar/coco).
+- The format of the code for evaluation is based on [MSCOCO evaluation code](https://github.com/tylin/coco-caption).

minigpt4/common/vqa_tools/__init__.py

@@ -0,0 +1,8 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+__author__ = "aagrawal"

minigpt4/common/vqa_tools/aokvqa/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022 Allen Institute for Artificial Intelligence
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

minigpt4/common/vqa_tools/aokvqa/README.md

@@ -0,0 +1,207 @@
+# A-OKVQA
+
+Official repository for **A-OKVQA: A Benchmark for Visual Question Answering using World Knowledge**.
+
+Links: [[Paper]](https://arxiv.org/abs/2206.01718) [[Website]](http://a-okvqa.allenai.org) [[Leaderboard]](https://leaderboard.allenai.org/a-okvqa/submissions/public)
+
+### Abstract
+
+The Visual Question Answering (VQA) task aspires to provide a meaningful testbed for the development of AI models that can jointly reason over visual and natural language inputs. Despite a proliferation of VQA datasets, this goal is hindered by a set of common limitations. These include a reliance on relatively simplistic questions that are repetitive in both concepts and linguistic structure, little world knowledge needed outside of the paired image, and limited reasoning required to arrive at the correct answer. We introduce A-OKVQA, a crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer. In contrast to the existing knowledge-based VQA datasets, the questions generally cannot be answered by simply querying a knowledge base, and instead require some form of commonsense reasoning about the scene depicted in the image.  We demonstrate the potential of this new dataset through a detailed analysis of its contents and baseline performance measurements over a variety of state-of-the-art vision–language models.
+
+![dataset_web](https://user-images.githubusercontent.com/28768645/170799740-f0d9ea60-6aff-4322-98d5-cae8e05983f4.svg)
+
+<hr>
+
+#### Table of Contents
+
+- [Getting started](#getting-started)
+  * [Downloading the dataset](#downloading-the-dataset)
+- [Evaluation & Leaderboard](#evaluation)
+- [Codebase](#codebase)
+  * [Preparing data](#preparing-data)
+  * [Models and Predictions](#models-and-predictions)
+
+<hr>
+
+## Getting started
+
+```bash
+git clone --single-branch --recurse-submodules https://github.com/allenai/aokvqa.git
+
+cd aokvqa
+export PYTHONPATH=.
+
+conda env create --name aokvqa
+conda activate aokvqa
+```
+
+### Downloading the dataset
+
+```bash
+export AOKVQA_DIR=./datasets/aokvqa/
+mkdir -p ${AOKVQA_DIR}
+
+curl -fsSL https://prior-datasets.s3.us-east-2.amazonaws.com/aokvqa/aokvqa_v1p0.tar.gz | tar xvz -C ${AOKVQA_DIR}
+```
+
+<details> <summary><b>Downloading COCO 2017</b></summary>
+
+```bash
+export COCO_DIR=./datasets/coco/
+mkdir -p ${COCO_DIR}
+
+for split in train val test; do
+    wget "http://images.cocodataset.org/zips/${split}2017.zip"
+    unzip "${split}2017.zip" -d ${COCO_DIR}; rm "${split}2017.zip"
+done
+
+wget http://images.cocodataset.org/annotations/annotations_trainval2017.zip
+unzip annotations_trainval2017.zip -d ${COCO_DIR}; rm annotations_trainval2017.zip
+```
+
+</details>
+
+Loading our dataset is easy! Just grab our [load_aokvqa.py](https://github.com/allenai/aokvqa/blob/main/load_aokvqa.py) file and refer to the following code.
+
+```python
+import os
+aokvqa_dir = os.getenv('AOKVQA_DIR')
+
+from load_aokvqa import load_aokvqa, get_coco_path
+train_dataset = load_aokvqa(aokvqa_dir, 'train')  # also 'val' or 'test'
+```
+
+<details> <summary><b>Example dataset entry</b></summary>
+
+```python
+dataset_example = train_dataset[0]
+
+print(dataset_example['question_id'])
+# 22MexNkBPpdZGX6sxbxVBH
+
+coco_dir = os.getenv('COCO_DIR')
+image_path = get_coco_path('train', dataset_example['image_id'], coco_dir)
+print(image_path)
+# ./datasets/coco/train2017/000000299207.jpg
+
+print(dataset_example['question'])
+print(dataset_example['choices'])
+# What is the man by the bags awaiting?
+# ['skateboarder', 'train', 'delivery', 'cab']
+
+correct_choice = dataset_example['choices'][ dataset_example['correct_choice_idx'] ]
+# Corrrect: cab
+
+print(dataset_example['rationales'][0])
+# A train would not be on the street, he would not have luggage waiting for a delivery, and the skateboarder is there and not paying attention to him so a cab is the only possible answer.
+```
+
+</details>
+
+## Evaluation
+
+Please prepare `predictions_{split}.json` files (for `split: {val,test}`) in the format below. You may omit either `multiple_choice` or `direct_answer` field if you only want to evaluate one setting.
+
+```python
+{
+    '<question_id>' : {
+        'multiple_choice' : '<prediction>',
+        'direct_answer' : '<prediction>'
+    }
+}
+```
+
+You can run evaluation on the validation set as follows.
+
+```bash
+python evaluation/eval_predictions.py --aokvqa-dir ${AOKVQA_DIR} --split val --preds ./predictions_val.json
+```
+
+### Leaderboard
+
+You may submit `predictions_test.json` to the [leaderboard](https://leaderboard.allenai.org/a-okvqa/submissions/get-started).
+
+## Codebase
+
+We provide all code and pretrained models necessary to replicate our experiments for Large-Scale Pretrained Models (sec. 5.2) and Rationale Generation (sec. 5.3).
+
+### Preparing data
+
+```bash
+export FEATURES_DIR=./features/
+mkdir -p ${FEATURES_DIR}
+```
+
+You can compute CLIP features for our vocabulary and dataset. These are most commonly used by our other experiments.
+
+```bash
+python data_scripts/encode_vocab_clip.py --vocab ${AOKVQA_DIR}/large_vocab_train.csv --model-type ViT-B/32 --out ${FEATURES_DIR}/clip-ViT-B-32_large_vocab.pt
+
+for split in train val test; do
+    python data_scripts/extract_clip_features.py --aokvqa-dir ${AOKVQA_DIR} --coco-dir ${COCO_DIR} --split ${split} --model-type ViT-B/32 --out ${FEATURES_DIR}/clip-ViT-B-32_${split}.pt
+done
+```
+
+<details> <summary><b>For training ClipCap with a transformer mapping network</b></summary>
+
+If you want to train our ClipCap models with the transformer mapping network (instead of an MLP, like we do), you'll also need to run `extract_clip_features.py` with `--model-type RN50x4`.
+
+</details>
+
+<details> <summary><b>For ResNet and BERT input features</b></summary>
+
+Our ResNet and BERT classification experiments require these respective features instead of CLIP. To generate these, please run the following commands:
+
+```bash
+# ResNet
+for split in train val test; do
+    python data_scripts/extract_resnet_features.py --aokvqa-dir ${AOKVQA_DIR} --coco-dir ${COCO_DIR} --split ${split} --out ${FEATURES_DIR}/resnet_${split}.pt
+done
+
+# BERT
+for split in train val test; do
+    python data_scripts/extract_bert_features.py --aokvqa-dir ${AOKVQA_DIR} --split ${split} --out ${FEATURES_DIR}/bert_${split}.pt
+done
+```
+
+</details>
+
+### Models and Predictions
+
+```bash
+export LOG_DIR=./logs/
+export PREDS_DIR=./predictions/
+export PT_MODEL_DIR=./pretrained_models/
+mkdir -p ${LOG_DIR} ${PREDS_DIR} ${PT_MODEL_DIR}
+```
+
+<details> <summary><b>Download our pretrained model weights</b></summary>
+
+```bash
+# Checkpoints for transfer learning experiments
+curl -fsSL https://prior-model-weights.s3.us-east-2.amazonaws.com/aokvqa/transfer_exp_checkpoints.tar.gz | tar xvz -C ${PT_MODEL_DIR}/aokvqa_models
+
+# Checkpoints for ClipCap models (generating answers and rationales)
+curl -fsSL https://prior-model-weights.s3.us-east-2.amazonaws.com/aokvqa/clipcap_checkpoints.tar.gz | tar xvz -C ${PT_MODEL_DIR}/aokvqa_models
+```
+
+</details>
+
+We have included instructions for replicating each of our experiments (see README.md files below).
+
+All Python scripts should be run from the root of this repository. Please be sure to first run the installation and data preparation as directed above.
+
+- [Heuristics](./heuristics/README.md)
+- [Transfer Learning Experiments](./transfer_experiments/README.md)
+- [Querying GPT-3](./gpt3/README.md)
+- [ClipCap](https://github.com/allenai/aokvqa/blob/ClipCap/README.md)
+- [Generating Captions & Rationales](https://github.com/allenai/aokvqa/blob/ClipCap/README.md)
+
+For each experiment, we follow this prediction file naming scheme: `{model-name}_{split}-{setting}.json` (e.g. `random-weighted_val-mc.json` or `random-weighted_test-da.json`). As examples in these Readme files, we produce predictions on the validation set.
+
+We unify predictions for each split before evaluation. (You can omit one of `--mc` or `--da` prediction file if you only want to evaluate one setting.)
+
+```bash
+python evaluation/prepare_predictions.py --aokvqa-dir ${AOKVQA_DIR} --split val --mc ./predictions_val-mc.json --da ./predictions_val-da.json --out ./predictions_val.json
+# repeat for test split ...
+```

minigpt4/common/vqa_tools/aokvqa/data_scripts/build_vocab.py

@@ -0,0 +1,45 @@
+import os
+import argparse
+from collections import Counter
+import pathlib
+
+from load_aokvqa import load_aokvqa
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--out', type=pathlib.Path, required=True, dest='output_file')
+args = parser.parse_args()
+
+
+# Build vocab from train set: correct choices + (direct answers appearing in >= 3 )
+
+train_set = load_aokvqa(args.aokvqa_dir, 'train')
+
+vocab = []
+all_choices = Counter()
+direct_answers = Counter()
+
+for i in train_set:
+    vocab.append( i['choices'][i['correct_choice_idx']] )
+    all_choices.update(i['choices'])
+    direct_answers.update(set(i['direct_answers']))
+vocab += [k for k,v in all_choices.items() if v >= 3]
+vocab += [k for k,v in direct_answers.items() if v >= 3]
+
+vocab = sorted(set(vocab))
+print(f"Vocab size: {len(vocab)}")
+
+# Save vocabulary Output
+
+with open(args.output_file, 'w') as f:
+    for v in vocab:
+        print(v, file=f)
+
+## Check validation set coverage
+
+val_set = load_aokvqa(args.aokvqa_dir, 'val')
+
+val_acc = [v['choices'][v['correct_choice_idx']] in vocab for v in val_set]
+val_acc = sum(val_acc) / len(val_acc) * 100
+print(f"Val set coverage: {val_acc:.2f}" )

minigpt4/common/vqa_tools/aokvqa/data_scripts/encode_vocab_clip.py

@@ -0,0 +1,26 @@
+import json
+from tqdm import tqdm
+import argparse
+import pathlib
+
+import torch
+import clip
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--vocab', type=pathlib.Path, required=True, dest='vocab_file')
+parser.add_argument('--model-type', type=str, choices=['RN50', 'RN50x4', 'RN50x16', 'RN50x64', 'RN101', 'ViT-B/32', 'ViT-B/16', 'ViT-L/14', 'ViT-L/14@336px'], required=True, dest='model_type')
+parser.add_argument('--out', type=pathlib.Path, required=True, dest='output_file')
+args = parser.parse_args()
+
+assert args.output_file.suffix == '.pt'
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model, preprocess = clip.load(args.model_type, device=device)
+
+with torch.no_grad():
+    a = open(args.vocab_file).read().splitlines()
+    mc_text = clip.tokenize(a).to(device)
+    mc_text_features = torch.stack([model.encode_text(mct.unsqueeze(0)).cpu() for mct in tqdm(mc_text)], dim=1)[0]
+    mc_text_features = mc_text_features.float()
+    model_name = args.model_type.replace('/', '-').replace('@', '-')
+    torch.save(mc_text_features, args.output_file)

minigpt4/common/vqa_tools/aokvqa/data_scripts/extract_bert_features.py

@@ -0,0 +1,50 @@
+import os
+import argparse
+import pathlib
+from tqdm import tqdm
+
+import torch
+from transformers import AutoTokenizer, AutoModel
+
+from load_aokvqa import load_aokvqa
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+parser.add_argument('--out', type=pathlib.Path, required=True, dest='output_file')
+args = parser.parse_args()
+
+assert args.output_file.suffix == '.pt'
+
+## Load dataset
+
+dataset = load_aokvqa(args.aokvqa_dir, args.split)
+
+## Load model
+
+tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens')
+model = AutoModel.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens')
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = model.to(device)
+model.eval()
+
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] # First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+## Encoding loop
+
+with torch.no_grad():
+    embeddings = {}
+
+    for d in tqdm(dataset):
+        encoded_input = tokenizer([d['question']], padding=True, return_tensors='pt')
+        encoded_input = {k:v.to(device) for k,v in encoded_input.items()}
+        e = mean_pooling(model(**encoded_input), encoded_input['attention_mask'])
+        embeddings[d['question_id']] = {
+            'question' : e[0].cpu()
+        }
+
+    torch.save(embeddings, args.output_file)

minigpt4/common/vqa_tools/aokvqa/data_scripts/extract_clip_features.py

@@ -0,0 +1,51 @@
+import os
+from PIL import Image
+from tqdm import tqdm
+import argparse
+import pathlib
+
+import torch
+import clip
+
+from load_aokvqa import load_aokvqa, get_coco_path
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--coco-dir', type=pathlib.Path, required=True, dest='coco_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+parser.add_argument('--model-type', type=str, choices=['RN50', 'RN50x4', 'RN50x16', 'RN50x64', 'RN101', 'ViT-B/32', 'ViT-B/16', 'ViT-L/14', 'ViT-L/14@336px'], required=True, dest='model_type')
+parser.add_argument('--out', type=pathlib.Path, required=True, dest='output_file')
+args = parser.parse_args()
+
+assert args.output_file.suffix == '.pt'
+
+## Load dataset
+
+dataset = load_aokvqa(args.aokvqa_dir, args.split)
+
+## Load model
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model, preprocess = clip.load(args.model_type, device=device)
+
+## Encoding loop
+
+with torch.no_grad():
+    embeddings = {}
+
+    for d in tqdm(dataset):
+        q = d["question"]
+        q_text = clip.tokenize(q).to(device)
+        q_text_features = model.encode_text(q_text)
+
+        img = Image.open(get_coco_path(args.split, d['image_id'], args.coco_dir))
+        img = preprocess(img).unsqueeze(0).to(device)
+        image_features = model.encode_image(img)
+
+        embeddings[d['question_id']] = {
+            'question' : q_text_features[0].float().cpu(),
+            'image' : image_features[0].float().cpu(),
+        }
+
+    torch.save(embeddings, args.output_file)

minigpt4/common/vqa_tools/aokvqa/data_scripts/extract_resnet_features.py

@@ -0,0 +1,62 @@
+import os
+import argparse
+import pathlib
+from tqdm import tqdm
+from PIL import Image
+
+import torch
+import torch.nn as nn
+from torchvision import models
+from torchvision import transforms as T
+
+from load_aokvqa import load_aokvqa, get_coco_path
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--coco-dir', type=pathlib.Path, required=True, dest='coco_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+parser.add_argument('--out', type=pathlib.Path, required=True, dest='output_file')
+args = parser.parse_args()
+
+assert args.output_file.suffix == '.pt'
+
+## Load dataset
+
+dataset = load_aokvqa(args.aokvqa_dir, args.split)
+
+## Load model
+
+resnet_preprocess = T.Compose([
+    T.Resize(size=224, interpolation=T.InterpolationMode.BICUBIC),
+    T.CenterCrop(size=(224, 224)),
+    T.ToTensor(),
+    T.Normalize(
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]
+    )
+])
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+resnet_model = models.resnet50(pretrained=True)
+resnet_model = torch.nn.Sequential(
+    *list(resnet_model.children())[:-1],
+    nn.Flatten()
+)  # strip classification layer
+resnet_model = resnet_model.to(device)
+
+## Encoding loop
+
+with torch.no_grad():
+    embeddings = {}
+
+    for d in tqdm(dataset):
+        img = Image.open(get_coco_path(args.split, d['image_id'], args.coco_dir)).convert('RGB')
+        resnet_input = resnet_preprocess(img).unsqueeze(0).to(device)
+        resnet_features = resnet_model(resnet_input)
+        embeddings[d['question_id']] = {
+            'image' : resnet_features[0].cpu()
+        }
+
+    torch.save(embeddings, args.output_file)

minigpt4/common/vqa_tools/aokvqa/environment.yml

@@ -0,0 +1,36 @@
+name: aokvqa
+channels:
+  - pytorch
+  - nvidia
+  - huggingface
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.7
+  - cudatoolkit=11.3
+  - numpy=1.21.6
+  - pytorch=1.11.0
+  - torchvision=0.12.0
+  - pytorch-lightning=1.6.3
+  - torchmetrics=0.8.1
+  - gdown=4.4.0
+  - pip=22.0.4
+  - pip:
+    - argparse==1.4.0
+    - Pillow==9.0.1
+    - tensorboard==2.9.0
+    - ftfy==6.1.1
+    - regex==2022.3.15
+    - tqdm==4.64.0
+    - clip @ git+https://github.com/openai/CLIP.git@b46f5ac7587d2e1862f8b7b1573179d80dcdd620
+    - openai==0.18.1
+    - nltk==3.7
+    - sacrebleu==2.0.0
+    - sacremoses==0.0.53
+    - sentence-transformers==2.2.0
+    - datasets==2.1.0
+    - tokenizers==0.10.3
+    - transformers==4.10.3
+
+# Next: resolve conflict between sentence-transfomers and pytorch-lightning
+# pip uninstall sentencepiece

minigpt4/common/vqa_tools/aokvqa/evaluation/eval_predictions.py

@@ -0,0 +1,97 @@
+import argparse
+import pathlib
+import json
+import glob
+
+from load_aokvqa import load_aokvqa
+
+
+def eval_aokvqa(dataset, preds, multiple_choice=False, strict=True):
+
+    if isinstance(dataset, list):
+        dataset = { dataset[i]['question_id'] : dataset[i] for i in range(len(dataset)) }
+
+    if multiple_choice is False:
+        dataset = {k:v for k,v in dataset.items() if v['difficult_direct_answer'] is False}
+
+    if strict:
+        dataset_qids = set(dataset.keys())
+        preds_qids = set(preds.keys())
+        assert dataset_qids.issubset(preds_qids)
+
+    # dataset = q_id (str) : dataset element (dict)
+    # preds = q_id (str) : prediction (str)
+
+    acc = []
+
+    for q in dataset.keys():
+        if q not in preds.keys():
+            acc.append(0.0)
+            continue
+
+        pred = preds[q]
+        choices = dataset[q]['choices']
+        direct_answers = dataset[q]['direct_answers']
+
+        ## Multiple Choice setting
+        if multiple_choice:
+            if strict:
+                assert pred in choices, 'Prediction must be a valid choice'
+            correct_choice_idx = dataset[q]['correct_choice_idx']
+            acc.append( float(pred == choices[correct_choice_idx]) )
+        ## Direct Answer setting
+        else:
+            num_match = sum([pred.lower() == da.lower() for da in direct_answers])
+            vqa_acc = min(1.0, num_match / 3.0)
+            acc.append(vqa_acc)
+
+    acc = sum(acc) / len(acc) * 100
+
+    return acc
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+    parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+    parser.add_argument('--preds', type=str, required=True, dest='prediction_files')
+    args = parser.parse_args()
+
+    dataset = load_aokvqa(args.aokvqa_dir, args.split)
+
+    for prediction_file in glob.glob(args.prediction_files):
+        predictions = json.load(open(prediction_file, 'r'))
+
+        # Multiple choice
+
+        mc_predictions = {}
+
+        for q in predictions.keys():
+            if 'multiple_choice' in predictions[q].keys():
+                mc_predictions[q] = predictions[q]['multiple_choice']
+
+        if mc_predictions != {}:
+            mc_acc = eval_aokvqa(
+                dataset,
+                mc_predictions,
+                multiple_choice=True,
+                strict=False
+            )
+            print(prediction_file, 'MC', mc_acc)
+
+        # Direct Answer
+
+        da_predictions = {}
+
+        for q in predictions.keys():
+            if 'direct_answer' in predictions[q].keys():
+                da_predictions[q] = predictions[q]['direct_answer']
+
+        if da_predictions != {}:
+            da_acc = eval_aokvqa(
+                dataset,
+                da_predictions,
+                multiple_choice=False,
+                strict=False
+            )
+            print(prediction_file, 'DA', da_acc)

minigpt4/common/vqa_tools/aokvqa/evaluation/load_aokvqa.py

@@ -0,0 +1,13 @@
+import os
+import json
+
+
+def load_aokvqa(aokvqa_dir, split, version='v1p0'):
+    assert split in ['train', 'val', 'test', 'test_w_ans']
+    dataset = json.load(open(
+        os.path.join(aokvqa_dir, f"aokvqa_{version}_{split}.json")
+    ))
+    return dataset
+
+def get_coco_path(split, image_id, coco_dir):
+    return os.path.join(coco_dir, f"{split}2017", f"{image_id:012}.jpg")

minigpt4/common/vqa_tools/aokvqa/evaluation/prepare_predictions.py

@@ -0,0 +1,31 @@
+import argparse
+import pathlib
+import json
+
+from load_aokvqa import load_aokvqa
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+    parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+    parser.add_argument('--mc', type=argparse.FileType('r'), dest='mc_pred_file')
+    parser.add_argument('--da', type=argparse.FileType('r'), dest='da_pred_file')
+    parser.add_argument('--out', type=argparse.FileType('w'), dest='output_file')
+    args = parser.parse_args()
+    assert args.mc_pred_file or args.da_pred_file
+
+    dataset = load_aokvqa(args.aokvqa_dir, args.split)
+    mc_preds = json.load(args.mc_pred_file) if args.mc_pred_file else None
+    da_preds = json.load(args.da_pred_file) if args.da_pred_file else None
+    predictions = {}
+
+    for d in dataset:
+        q = d['question_id']
+        predictions[q] = {}
+        if mc_preds and q in mc_preds.keys():
+            predictions[q]['multiple_choice'] = mc_preds[q]
+        if da_preds and q in da_preds.keys():
+            predictions[q]['direct_answer'] = da_preds[q]
+
+    json.dump(predictions, args.output_file)

minigpt4/common/vqa_tools/aokvqa/evaluation/remap_predictions.py

@@ -0,0 +1,44 @@
+import argparse
+import pathlib
+import json
+from tqdm import tqdm
+
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import cos_sim
+
+from load_aokvqa import load_aokvqa
+
+
+def map_to_choices(dataset, predictions, device='cpu'):
+    if isinstance(dataset, list):
+        dataset = { dataset[i]['question_id'] : dataset[i] for i in range(len(dataset)) }
+
+    if all([p in dataset[q]['choices'] for q, p in predictions.items()]):
+        return predictions
+
+    model = SentenceTransformer('sentence-transformers/average_word_embeddings_glove.6B.300d')
+    model.to(device)
+    for q in tqdm(predictions.keys()):
+        choices = dataset[q]['choices']
+        if predictions[q] not in choices:
+            choice_embeddings = model.encode([predictions[q]] + choices, convert_to_tensor=True)
+            a_idx = cos_sim(choice_embeddings[0], choice_embeddings[1:]).argmax().item()
+            predictions[q] = choices[a_idx]
+
+    return predictions
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+    parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+    parser.add_argument('--pred', type=argparse.FileType('r'), required=True, dest='prediction_file')
+    parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+    args = parser.parse_args()
+
+
+    dataset = load_aokvqa(args.aokvqa_dir, args.split)
+    predictions = json.load(args.prediction_file)
+    predictions = map_to_choices(dataset, predictions)
+
+    json.dump(predictions, args.output_file)

minigpt4/common/vqa_tools/aokvqa/gpt3/README.md

@@ -0,0 +1,14 @@
+## Querying GPT-3
+
+To follow our experiments which use GPT-3, you must have access to the [OpenAI API](https://openai.com/api/) (at cost). Please retrieve your [organization](https://beta.openai.com/account/org-settings) and [API](https://beta.openai.com/account/api-keys) keys and set them in your environment variables.
+
+```bash
+export OPENAI_ORG=....
+export OPENAI_API_KEY=...
+```
+
+For producing predictions for both DA and MC settings, run:
+```bash
+python gpt3/query_gpt3.py --aokvqa-dir ${AOKVQA_DIR} --split val --out ${PREDS_DIR}/gpt3_val-da.json
+python remap_predictions.py --aokvqa-dir ${AOKVQA_DIR} --split val --pred ${PREDS_DIR}/gpt3_val-da.json --out ${PREDS_DIR}/gpt3_val-mc.json
+```

minigpt4/common/vqa_tools/aokvqa/gpt3/caption_inputs.py

@@ -0,0 +1,23 @@
+import os
+import json
+import argparse
+import pathlib
+
+from load_aokvqa import load_aokvqa
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--coco-dir', type=pathlib.Path, required=True, dest='coco_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val'], required=True)
+parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+args = parser.parse_args()
+
+aokvqa_set = load_aokvqa(args.aokvqa_dir, args.split)
+
+coco_captions = json.load(open(os.path.join(args.coco_dir, 'annotations', f'captions_{args.split}2017.json')))['annotations']
+coco_captions = {c['image_id'] : c['caption'] for c in coco_captions}
+
+captions = { d['question_id'] : coco_captions[d['image_id']] for d in aokvqa_set }
+
+json.dump(captions, args.output_file)

minigpt4/common/vqa_tools/aokvqa/gpt3/query_gpt3.py

@@ -0,0 +1,79 @@
+import os
+import random
+import json
+from tqdm import tqdm
+import argparse
+import pathlib
+
+import openai
+openai.organization = os.getenv('OPENAI_ORG')
+openai.api_key = os.getenv('OPENAI_API_KEY')
+
+from load_aokvqa import load_aokvqa
+
+
+random.seed(0)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+    parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+    parser.add_argument('--n', type=int, default=10, dest='num_examples')
+    parser.add_argument('--train-context', type=argparse.FileType('r'), dest='train_context_file')
+    parser.add_argument('--prefix', type=str, default='', dest='prompt_prefix')
+    parser.add_argument('--include-choices', action='store_true', dest='include_choices')
+    parser.add_argument('--context', type=argparse.FileType('r'), dest='context_file')
+    parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+    args = parser.parse_args()
+
+
+    train_set = load_aokvqa(args.aokvqa_dir, 'train')
+    eval_set = load_aokvqa(args.aokvqa_dir, args.split)
+
+    train_context = {}
+    context = {}
+    if args.context_file is not None:
+        train_context = json.load(args.train_context_file)
+        context = json.load(args.context_file)
+
+    predictions = {}
+
+    for d in tqdm(eval_set):
+        q = d['question_id']
+
+        prompt = args.prompt_prefix
+        for e in random.sample(train_set, args.num_examples):
+            prompt += prompt_element(e,
+                context=train_context.get(q, None),
+                include_choices=args.include_choices,
+                answer=True
+            )
+            prompt += '\n\n'
+
+        prompt += prompt_element(d,
+            context=context.get(q, None),
+            include_choices=args.include_choices,
+            answer=False
+        )
+
+        response = openai.Completion.create(
+            engine="text-curie-001",
+            prompt=prompt,
+            temperature=0.0,
+            max_tokens=10,
+        )
+
+        predictions[q] = response.choices[0].text.strip()
+
+    json.dump(predictions, args.output_file)
+
+
+def prompt_element(d, context=None, include_choices=False, answer=False):
+    return (f"Context: {context}\n" if context is not None else '') + \
+            f"Q: {d['question']}\n" + \
+           (f"Choices: {', '.join(d['choices'])}.\n" if include_choices else '') + \
+            f"A:" + (f" {d['choices'][d['correct_choice_idx']]}" if answer else '')
+
+if __name__ == '__main__':
+    main()

minigpt4/common/vqa_tools/aokvqa/gpt3/rationale_inputs.py

@@ -0,0 +1,16 @@
+import json
+import argparse
+import pathlib
+
+from load_aokvqa import load_aokvqa
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val', 'test_w_ans'], required=True)
+parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+args = parser.parse_args()
+
+aokvqa_set = load_aokvqa(args.aokvqa_dir, args.split)
+rationales = {d['question_id'] : d['rationales'][0] for d in aokvqa_set}
+json.dump(rationales, args.output_file)