Hugging Face's logo

Spaces:
Vchitect
/
VBench
Running

App Files Community
1
VBench
File size: 3,940 Bytes 
bf9e163
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11eaafd
bf9e163
 
11eaafd
bf9e163
 
 
 
 
 
 
 
 
b374405
 
bf9e163
 
 
 
11eaafd
 
 
bf9e163
 
 
 
 
 
 
 
 
 
 
 
 
11eaafd
bf9e163
 
 
11eaafd
bf9e163
 
 
 
 
 
 
 
 
 
 
 
 
 
7b532c9
bf9e163
7b532c9
bf9e163
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
import streamlit as st
st.set_page_config(layout="wide")
import pandas  as pd
import os
import json
import shutil
from huggingface_hub import Repository

REFERENCE_NAME = "references"
SUBMISSION_NAME = "vbench_leaderboard_submission"


SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/VBench/", SUBMISSION_NAME)


TEST_SETS = [
    "subject consistency",
    "background consistency",
    "temporal flickering",
    "motion smoothness",
    "dynamic degree",
    "aesthetic quality",
    "imaging quality",
    "object class",
    "multiple objects",
    "human action",
    "color",
    "spatial relationship",
    "scene",
    "appearance style",
    "temporal style",
    "overall consistency"
    ]

style = """
<style>
    th {
        font-size: 10px;
    }
</style>
"""

CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv")

HF_TOKEN = os.environ.get("HF_TOKEN")

try:
    submission_repo = Repository(
        local_dir="vbench_leaderboard_submission", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset"
    )
    submission_repo.git_pull()
except Exception as e:
    print(e)


all_submissions = [
    file_name
    for file_name in os.listdir(SUBMISSION_NAME)
    if file_name.endswith('.json')
]

all_results = pd.read_csv(CSV_RESULTS_FILE)
with open(os.path.join(SUBMISSION_NAME, "verified_model.txt")) as f:
    verified_model = [i.strip().lower() for i in f.readlines()]
all_results['verified'] = all_results['name'].apply(lambda x: '√' if x.lower() in verified_model else ' ')


# Write table form CSV
table = all_results.copy()
table = table.round(4)
# columns_to_convert = table.columns[1:-1]
# table[columns_to_convert] = table[columns_to_convert].applymap(lambda x: f'{x * 100}'[:5]+'%')



# Streamlit
st.markdown("# VBench ")


st.markdown(
    f"""
    This is the leaderboard of VBench: Comprehensive Benchmark Suite for Video Generative Models (VBench).
    """
)

default_index = list(table.columns[1:-1]).index("overall consistency") if "overall consistency" in table.columns else 0
sort_option = st.selectbox(
    'Choose a column to sort by',
    table.columns[1:-1], 
    index=default_index,
)
table = table.sort_values(by=sort_option, ascending=False)

st.write(style + table.to_markdown(index=False), unsafe_allow_html=True)
st.markdown(
    """
    For more information, refer to the paper submission on [Arxiv](https://).
    """
)

st.markdown(
    """
    ## Submitting to VBench
    \n
    To submit to VBench, download the prompt suite from [VBench/Prompt](https://github.com/Vchitect/VBench/tree/master/prompts). Upload your zipped submissions for scoring and placement on the leaderboard. 
    \n
    Should you experience any issues, open an issue using the link [new discussion](https://huggingface.co/spaces/VBench/T2V-Leaderboard/discussions) and tag `@Ziqi`.
 """
)

# Using the "with" syntax
with st.form(key="my_form"):
    uploaded_file = st.file_uploader("Choose a json file")
    submit_button = st.form_submit_button(label="Submit")

if submit_button:
    if uploaded_file is None:
        raise ValueError("Please make sure to have uploaded a json file.")

    submission = uploaded_file.name.split(".json")[0]
    with st.spinner(f"Uploading {submission}..."):
        with open(os.path.join(submission_repo.local_dir, os.path.basename(uploaded_file.name)),'wb') as f:
            f.write(uploaded_file.getvalue())
        submission_repo.push_to_hub()

    with st.spinner(f"Update Score for {submission}..."):
        results = {"name": submission}
        upload_score = json.loads(uploaded_file.getvalue())
        for info in upload_score:
            results[info['dimension']] = info['final_score']
        all_results.loc[len(all_results)] = results
        all_results.to_csv(CSV_RESULTS_FILE, index=False)
        commit_url = submission_repo.push_to_hub()

    st.success('Please refresh this space (CTRL+R) to see your result')