Initial commit

app.py

@@ -0,0 +1,121 @@
+"""
+Adapted from the SEED-Bench Leaderboard by AILab-CVC
+Source: https://huggingface.co/spaces/AILab-CVC/SEED-Bench_Leaderboard
+"""
+
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
+import gradio as gr
+import pandas as pd
+import json
+import pdb
+import tempfile
+
+from constants import *
+from src.auto_leaderboard.model_metadata_type import ModelType
+
+global data_component, filter_component
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+def get_baseline_df():
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Final Sum Score", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    df = df[present_columns]
+    print(df)
+    return df
+
+def get_all_df():
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Final Sum Score", ascending=False)
+    print(df)
+    return df
+
+block = gr.Blocks()
+
+
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 EvalCrafter Benchmark", elem_id="evalcrafter-benchmark-tab-table", id=0):
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO_v2,
+                value=AVG_INFO,
+                label="Select options",
+                interactive=True,
+            )
+
+            # 创建数据帧组件
+            # pdb.set_trace()
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            def on_checkbox_group_change(selected_columns):
+                # pdb.set_trace()
+                selected_columns = [item for item in TASK_INFO_v2 if item in selected_columns]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_all_df()[present_columns]
+                updated_data = updated_data.sort_values(by=present_columns[3], ascending=False)
+                updated_headers = present_columns
+                update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+                # pdb.set_trace()
+                filter_component = gr.components.Dataframe(
+                    value=updated_data, 
+                    headers=updated_headers,
+                    type="pandas", 
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                    )
+                # pdb.set_trace()
+                return filter_component.value
+
+            # 将复选框组关联到处理函数
+            checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component)
+
+    
+        # table 2
+        with gr.TabItem("📝 About", elem_id="evalcrafter-benchmark-tab-table", id=2):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+    
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            get_baseline_df, outputs=data_component
+        )
+
+    gr.Markdown(r"""
+        Please cite this paper if you find it useful ♥️:
+
+        ```bibtex
+        @inproceedings{Liu2023EvalCrafterBA,
+        title={EvalCrafter: Benchmarking and Evaluating Large Video Generation Models},
+        author={Yaofang Liu and Xiaodong Cun and Xuebo Liu and Xintao Wang and Yong Zhang and Haoxin Chen and Yang Liu and Tieyong Zeng and Raymond Chan and Ying Shan},
+        year={2023},
+        url={https://api.semanticscholar.org/CorpusID:264172222}
+        }
+        ```
+        """)
+    # block.load(get_baseline_df, outputs=data_title)
+
+block.launch(share=False)

constants.py

@@ -0,0 +1,47 @@
+# this is .py for store constants 
+MODEL_INFO = ['Models', 'Ver.','Abilities']
+TASK_INFO = [ 'Resolution', 'FPS', 'Open Source', 'Length', 'Speed', 'Motion', 'Camera', 'Final Sum Score', 'Motion Quality', 'Text-Video Alignment', 'Visual Quality', 'Temporal Consistency']
+TASK_INFO_v2 = ['Final Sum Score', 'Motion Quality', 'Text-Video Alignment', 'Visual Quality', 'Temporal Consistency', 'Resolution', 'FPS', 'Open Source', 'Length', 'Speed', 'Motion', 'Camera']
+
+AVG_INFO = ['Final Sum Score', 'Motion Quality', 'Text-Video Alignment', 'Visual Quality', 'Temporal Consistency']
+DATA_TITILE_TYPE = ["markdown", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"]
+CSV_DIR = "./file/result.csv"
+
+# COLUMN_NAMES = MODEL_INFO + TASK_INFO
+COLUMN_NAMES = MODEL_INFO + TASK_INFO_v2
+
+DATA_NUM = [3158, 1831, 4649, 978, 2447, 657, 97, 331, 85, 1740, 2077, 1192]
+
+
+LEADERBORAD_INTRODUCTION = """# EvalCrafter Leaderboard 🏆
+
+    Welcome to the cutting-edge leaderboard for text-to-video generation, where we meticulously evaluate state-of-the-art generative models using our comprehensive framework, ensuring high-quality results that align with user opinions. Join us in this exciting journey towards excellence! 🛫  
+
+    More methods will be evalcrafted soon, stay tunned ❤️ Join our evaluation by sending an email 📧 (vinthony@gmail.com)! You may also read the [EvalCrafter paper](https://arxiv.org/abs/2310.11440) for more detailed information 🤗
+    """
+
+TABLE_INTRODUCTION = """In the table below, we summarize each dimension performance of all the models.    """
+
+LEADERBORAD_INFO = """
+      The vision and language generative models have been overgrown in recent years. For video generation, 
+      various open-sourced models and public-available services are released for generating high-visual quality videos. 
+      However, these methods often use a few academic metrics, \eg, FVD or IS, to evaluate the performance. We argue that
+      it is hard to judge the large conditional generative models from the simple metrics since these models are often trained
+      on very large datasets with multi-aspect abilities. Thus, we propose a new framework and pipeline to exhaustively evaluate
+      the performance of the generated videos. To achieve this, we first conduct a new prompt list for text-to-video generation 
+      by analyzing the real-world prompt list with the help of the large language model. Then, we evaluate the state-of-the-art video 
+      generative models on our carefully designed benchmarks, in terms of visual qualities, content qualities, motion qualities, and 
+      text-caption alignment with around 18 objective metrics. To obtain the final leaderboard of the models, we also fit a series of
+      coefficients to align the objective metrics to the users' opinions. Based on the proposed opinion alignment method, our final score
+      shows a higher correlation than simply averaging the metrics, showing the effectiveness of the proposed evaluation method.
+"""
+
+
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@inproceedings{Liu2023EvalCrafterBA,
+  title={EvalCrafter: Benchmarking and Evaluating Large Video Generation Models},
+  author={Yaofang Liu and Xiaodong Cun and Xuebo Liu and Xintao Wang and Yong Zhang and Haoxin Chen and Yang Liu and Tieyong Zeng and Raymond Chan and Ying Shan},
+  year={2023},
+  url={https://api.semanticscholar.org/CorpusID:264172222}
+}"""

file/result.csv

@@ -0,0 +1,6 @@
+Models,Ver.,Abilities,Resolution,FPS,Open Source,Length,Speed,Motion,Camera,Final Sum Score,Motion Quality,Text-Video Alignment,Visual Quality,Temporal Consistency
+ModelScope-XL,23.08,I2V & V2V,1280x720,8,✓,4s,8 min+,-,-,221 ,59.41,47.22,55.23,59.31
+ZeroScope,23.06,T2V & V2V,1024x576,8,✓,4s,3 min,-,-,218 ,54.26,46.18,56.37,61.19
+Floor33 Pictures,23.08,T2V,1280x720,8,-,2s,4 min,-,-,219 ,51.97,51.29,59.53,56.36
+PikaLab,23.09,I2V OR T2V,1088x640,24,-,3s,1 min,✓,✓,245 ,57.74,54.11,63.52,69.35
+Gen2,23.09,I2V OR T2V,896x512,24,-,4s,1 min,✓,✓,252 ,62.53,52.3,67.35,69.71

file/result1.csv

@@ -0,0 +1,28 @@
+Model Type,Model,Language Model,Avg. All,Avg. Img,Avg. Video,Scene Understanding,Instance Identity,Instance Attributes,Instance Localization,Instance Counting,Spatial Relation,Instance Interaction,Visual Reasoning,Text Recognition,Action Recognition,Action Prediction,Procedure Understanding
+LLM,[Flan-T5](https://huggingface.co/google/flan-t5-xl),Flan-T5-XL,27.7,27.3,28.6,23,29,32.8,31.8,20.5,31.8,33,18.2,19.4,23.2,34.9,25.4
+LLM,[Vicuna](https://huggingface.co/lmsys/vicuna-7b-v1.3),Vicuna-7B,28.5,28.2,29.5,23.4,30.7,29.7,30.9,30.8,28.6,29.8,18.5,13.4,27.3,34.5,23.8
+LLM,[LLaMA](https://research.facebook.com/publications/llama-open-and-efficient-foundation-language-models/),LLaMA-7B,26.8,26.6,27.3,26.3,27.4,26.2,28.3,25.1,28.8,19.2,37,9,33,23.1,26.2
+ImageLLM,[BLIP-2](https://github.com/salesforce/LAVIS),Flan-T5-XL,46.4,49.7,36.7,59.1,53.9,49.2,42.3,43.2,36.7,55.7,45.6,25.9,32.6,47.5,24
+ImageLLM,[InstructBLIP](https://github.com/salesforce/LAVIS),Flan-T5-XL,52.7,57.8,38.3,60.3,58.5,63.4,40.6,58.4,38.7,51.6,45.9,25.9,33.1,49.1,27.1
+ImageLLM,[InstructBLIP-Vicuna](https://github.com/salesforce/LAVIS),Vicuna-7B,53.4,58.8,38.1,60.2,58.9,65.6,43.6,57.2,40.3,52.6,47.7,43.5,34.5,49.6,23.1
+ImageLLM,[LLaVA](https://github.com/haotian-liu/LLaVA),Vicuna-13B,61.6,68.2,42.7,74.9,71.3,68.9,63.5,61.3,51.4,73.2,77,60.5,48.9,41.1,36.6
+ImageLLM,[MiniGPT-4](https://github.com/Vision-CAIR/MiniGPT-4),Vicuna-7B,42.8,47.4,29.9,56.3,49.2,45.8,37.9,45.3,32.6,47.4,57.1,11.8,38.2,24.5,27.1
+ImageLLM,[VPGTrans](https://github.com/VPGTrans/VPGTrans),LLaMA-7B,39.1,41.8,31.4,51.9,44.1,39.9,36.1,33.7,36.4,32,53.2,30.6,39.5,24.3,31.9
+ImageLLM,[MultiModal-GPT](https://github.com/open-mmlab/Multimodal-GPT),LLaMA-7B,33.2,34.5,29.2,43.6,37.9,31.5,30.8,27.3,30.1,29.9,51.4,18.8,36.9,25.8,24
+ImageLLM,[Otter](https://github.com/Luodian/Otter),LLaMA-7B,33.9,35.2,30.4,44.9,38.6,32.2,30.9,26.3,31.8,32,51.4,31.8,37.9,27.2,24.8
+ImageLLM,[Otter](https://github.com/Luodian/Otter),MPT-7B,39.7,42.9,30.6,51.3,43.5,42.3,34.2,38.4,30.9,40.2,55.3,24.7,36.8,29.2,23.8
+ImageLLM,[OpenFlamingo](https://github.com/mlfoundations/open_flamingo),LLaMA-7B,33.1,34.5,29.3,43.9,38.1,31.3,30.1,27.3,30.6,29.9,50.2,20,37.2,25.4,24.2
+ImageLLM,[OpenFlamingo](https://github.com/mlfoundations/open_flamingo),MPT-7B,40.9,42.7,35.7,53.2,45.3,40,31.2,39.3,32.6,36.1,51.4,25.9,42.9,34.7,26.9
+ImageLLM,[LLaMA-AdapterV2](https://github.com/OpenGVLab/LLaMA-Adapter),LLaMA-7B,32.7,35.2,25.8,45.2,38.5,29.3,33,29.7,35.5,39.2,52,24.7,38.6,18.5,19.6
+ImageLLM,[GVT](https://github.com/TencentARC/GVT),Vicuna-7B,33.5,35.5,27.8,41.7,35.5,31.8,29.5,36.2,32,32,51.1,27.1,33.9,25.4,23
+ImageLLM,[mPLUG-Owl](https://github.com/X-PLUG/mPLUG-Owl),LLaMA-7B,34,37.9,23,49.7,45.3,32.5,36.7,27.3,32.7,44.3,54.7,28.8,26.7,17.9,26.5
+ImageLLM,[Kosmos-2](https://github.com/microsoft/unilm/tree/master/kosmos-2),Decoder Only 1.3B,50,54.4,37.5,63.4,57.1,58.5,44,41.4,37.9,55.7,60.7,25.9,41.3,40.4,27
+ImageLLM,[Qwen-VL-Chat](https://huggingface.co/Qwen/Qwen-VL-Chat),Qwen-7B,58.2,65.4,37.8,73.3,67.3,69.6,57.7,52.9,48.2,59.8,74.6,53.5,43.9,39.2,26.7
+ImageLLM,[Qwen-VL](https://huggingface.co/Qwen/Qwen-VL),Qwen-7B,56.3,62.3,39.1,71.2,66.4,67.7,53.5,44.8,43.8,62.9,74.9,51.2,44.7,38.5,32
+ImageLLM,[IDEFICS-9b-instruct](https://huggingface.co/HuggingFaceM4/idefics-9b-instruct),LLaMA-7B,0,44.5,0,55.8,45.3,42.3,40.2,36.8,34.9,37.1,55.9,38.8,0,0,0
+ImageLLM,[IDEFICS-80b-instruct](https://huggingface.co/HuggingFaceM4/idefics-9b-instruct),LLaMA-65B,0,53.2,0,64,52.6,50.8,48.3,46.1,45.5,62.9,68,51.8,0,0,0
+ImageLLM,[InternLM-XComposer-VL](https://github.com/InternLM/InternLM-XComposer),InternLM-7B,0,66.9,0,75,71.7,67.6,60.8,56.2,55.3,74.4,77,48.5,0,0,0
+ImageLLM,[SEED-LLaMA](https://github.com/AILab-CVC/SEED),LLaMA2-Chat-13b,48.9,53.7,35.4,64.1,54.2,54.1,46.5,45.3,38.2,51.6,60.7,44.7,37.8,45.3,20.0 
+VideoLLM,[VideoChat](https://github.com/OpenGVLab/Ask-Anything),Vicuna-7B,37.6,39,33.7,47.1,43.8,34.9,40,32.8,34.6,42.3,50.5,17.7,34.9,36.4,27.3
+VideoLLM,[Video-ChatGPT](https://github.com/mbzuai-oryx/Video-ChatGPT),LLaMA-7B,31.2,33.9,23.5,37.2,31.4,33.2,28.4,35.5,29.5,23.7,42.3,25.9,27.6,21.3,21.1
+VideoLLM,[Valley](https://github.com/RupertLuo/Valley),LLaMA-13B,30.3,32,25.4,39.3,32.9,31.6,27.9,24.2,30.1,27.8,43.8,11.8,31.3,23.2,20.7

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from enum import Enum
+import glob
+import json
+import os
+from typing import Dict, List
+
+from ..utils_display import AutoEvalColumn
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str # emoji
+
+model_type_symbols = {
+    "LLM": "🟢",
+    "ImageLLM": "🔶",
+    "VideoLLM": "⭕",
+    "Other": "🟦",
+}
+
+class ModelType(Enum):
+    PT = ModelInfo(name="LLM", symbol="🟢")
+    FT = ModelInfo(name="ImageLLM", symbol="🔶")
+    IFT = ModelInfo(name="VideoLLM", symbol="⭕")
+    RL = ModelInfo(name="Other", symbol="🟦")
+
+    def to_str(self, separator = " "):
+        return f"{self.value.symbol}{separator}{self.value.name}" 
+

src/utils_display.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+    hidden: bool = False
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True)
+    model = ColumnContent("Model", "markdown", True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC", "number", True)
+    hellaswag = ColumnContent("HellaSwag", "number", True)
+    mmlu = ColumnContent("MMLU", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False, True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "bool", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"