yinanhe commited on
Commit
bf9e163
β€’
1 Parent(s): a2b42ac
Files changed (3) hide show
  1. README.md +5 -5
  2. app.py +130 -0
  3. requirements.txt +3 -0
README.md CHANGED
@@ -1,13 +1,13 @@
1
  ---
2
- title: T2V Leaderboard
3
- emoji: πŸ†
4
  colorFrom: indigo
5
- colorTo: green
6
  sdk: streamlit
7
- sdk_version: 1.28.2
8
  app_file: app.py
9
  pinned: false
10
- license: mit
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: Leaderboard
3
+ emoji: πŸ”₯
4
  colorFrom: indigo
5
+ colorTo: pink
6
  sdk: streamlit
7
+ sdk_version: 1.26.0
8
  app_file: app.py
9
  pinned: false
10
+ license: cc-by-sa-4.0
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ st.set_page_config(layout="wide")
3
+ import pandas as pd
4
+ import os
5
+ import json
6
+ import shutil
7
+ from huggingface_hub import Repository
8
+
9
+ REFERENCE_NAME = "references"
10
+ SUBMISSION_NAME = "vbench_leaderboard_submission"
11
+
12
+
13
+ SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/VBench/", SUBMISSION_NAME)
14
+
15
+
16
+ TEST_SETS = [
17
+ "subject consistency",
18
+ "background consistency",
19
+ "temporal flickering",
20
+ "motion smoothness",
21
+ "dynamic degree",
22
+ "aesthetic quality",
23
+ "imaging quality",
24
+ "object class",
25
+ "multiple objects",
26
+ "human action",
27
+ "color",
28
+ "spatial relationship",
29
+ "scene",
30
+ "appearance style",
31
+ "temporal style",
32
+ "overall consistency"
33
+ ]
34
+
35
+ style = """
36
+ <style>
37
+ th {
38
+ font-size: 10px;
39
+ }
40
+ </style>
41
+ """
42
+
43
+ CSV_RESULTS_FILE = os.path.join(SUBMISSION_NAME, "results.csv")
44
+
45
+ HF_TOKEN = os.environ.get("HF_TOKEN")
46
+
47
+ try:
48
+ submission_repo = Repository(
49
+ local_dir="vbench_leaderboard_submission", clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset"
50
+ )
51
+ except Exception as e:
52
+ print(e)
53
+ submission_repo.git_pull()
54
+
55
+ all_submissions = [
56
+ file_name
57
+ for file_name in os.listdir(SUBMISSION_NAME)
58
+ if file_name.endswith('.json')
59
+ ]
60
+
61
+ all_results = pd.read_csv(CSV_RESULTS_FILE)
62
+ with open(os.path.join(SUBMISSION_NAME, "verified_model.txt")) as f:
63
+ verified_model = [i.strip() for i in f.readlines()]
64
+ all_results['verified'] = all_results['name'].apply(lambda x: '√' if x in verified_model else ' ')
65
+
66
+
67
+ # Write table form CSV
68
+ table = all_results.copy()
69
+ table = table.round(2)
70
+
71
+
72
+
73
+ # Streamlit
74
+ st.markdown("# VBench ")
75
+
76
+
77
+ st.markdown(
78
+ f"""
79
+ This is the leaderboard of VBench: Comprehensive Benchmark Suite for Video Generative Models (VBench).
80
+ """
81
+ )
82
+
83
+ sort_option = st.selectbox(
84
+ 'Choose a column to sort by',
85
+ table.columns[1:-1],
86
+ )
87
+ table = table.sort_values(by=sort_option, ascending=False)
88
+
89
+ st.write(style + table.to_markdown(index=False), unsafe_allow_html=True)
90
+ st.markdown(
91
+ """
92
+ For more information, refer to the paper submission on [Arxiv](https://).
93
+ """
94
+ )
95
+
96
+ st.markdown(
97
+ """
98
+ ## Submitting to VBench
99
+ \n
100
+ To submit to VBench, download the prompt suite from [VBench/Prompt](https://huggingface.co/datasets//). Upload your zipped submissions for scoring and placement on the leaderboard.
101
+ \n
102
+ Should you experience any issues, open an issue using the link [new discussion](http:) and tag `@vbench`.
103
+ """
104
+ )
105
+
106
+ # Using the "with" syntax
107
+ with st.form(key="my_form"):
108
+ uploaded_file = st.file_uploader("Choose a json file")
109
+ submit_button = st.form_submit_button(label="Submit")
110
+
111
+ if submit_button:
112
+ if uploaded_file is None:
113
+ raise ValueError("Please make sure to have uploaded a json file.")
114
+
115
+ submission = uploaded_file.name.split(".json")[0]
116
+ with st.spinner(f"Uploading {submission}..."):
117
+ with open(os.path.join(submission_repo.local_dir, os.path.basename(uploaded_file.name)),'wb') as f:
118
+ f.write(uploaded_file.getvalue())
119
+ submission_repo.push_to_hub()
120
+
121
+ with st.spinner(f"Update Score for {submission}..."):
122
+ results = {"name": submission}
123
+ upload_score = json.loads(uploaded_file.getvalue())
124
+ for info in upload_score:
125
+ results[info['dimension']] = info['final_score']
126
+ all_results.loc[len(all_results)] = results
127
+ all_results.to_csv(CSV_RESULTS_FILE, index=False)
128
+ commit_url = submission_repo.push_to_hub()
129
+
130
+ st.success('Please refresh this space (CTRL+R) to see your result')
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ huggingface_hub==0.8.1
2
+ pandas
3
+ streamlit