update

arena_elo/elo_rating/elo_analysis.py

@@ -171,20 +171,20 @@ def visualize_pairwise_win_fraction(battles, model_order):
         xaxis_side="top",
         title_y=0.07,
         title_x=0.5,
-        xaxis=dict(
-            tickfont=dict(size=16),
-            title=dict(font=dict(size=16)),
-        ),
-        yaxis=dict(
-            tickfont=dict(size=16),
-            title=dict(font=dict(size=16)),
-        ),
+        # xaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
+        # yaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
     )
     fig.update_traces(
-        textfont=dict(size=16),
-        colorbar=dict(
-            title=dict(font=dict(size=16))
-        ),
+        # textfont=dict(size=16),
+        # colorbar=dict(
+        #     title=dict(font=dict(size=16))
+        # ),
         hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Fraction of A Wins: %{z}<extra></extra>"
     )
 
@@ -208,8 +208,20 @@ def visualize_battle_count(battles, model_order):
         xaxis_side="top",
         title_y=0.07,
         title_x=0.5,
+        # xaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
+        # yaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
     )
     fig.update_traces(
+        # textfont=dict(size=16),
+        # colorbar=dict(
+        #     title=dict(font=dict(size=16))
+        # ),
         hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Count: %{z}<extra></extra>"
     )
     return fig
@@ -226,8 +238,17 @@ def visualize_average_win_rate(battles, limit_show_number):
         width=700,
     )
     fig.update_layout(
-        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False
+        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False,
+        # xaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
+        # yaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
     )
+    fig.update_traces(textfont_size=16)
     return fig
 
 
@@ -257,7 +278,17 @@ def visualize_bootstrap_elo_rating(df, df_final, limit_show_number):
         height=500,
         width=700,
     )
-    fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
+    fig.update_layout(xaxis_title="Model", yaxis_title="Rating",
+        # xaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
+        # yaxis=dict(
+        #     tickfont=dict(size=16),
+        #     title=dict(font=dict(size=16)),
+        # ),
+    )
+    fig.update_traces(textfont_size=16)
     return fig
 
 
@@ -380,13 +411,13 @@ if __name__ == "__main__":
     
     # # save heatmap results in the same directory of the cleaned battle file
     win_fraction_heatmap_file = args.clean_battle_file.replace(".json", "_win_fraction_heatmap.jpg")
-    # battle_count_heatmap_file = args.clean_battle_file.replace(".json", "_battle_count_heatmap.jpg")
-    # average_win_rate_bar_file = args.clean_battle_file.replace(".json", "_average_win_rate_bar.jpg")
-    # bootstrap_elo_rating_file = args.clean_battle_file.replace(".json", "_bootstrap_elo_rating.jpg")
+    battle_count_heatmap_file = args.clean_battle_file.replace(".json", "_battle_count_heatmap.jpg")
+    average_win_rate_bar_file = args.clean_battle_file.replace(".json", "_average_win_rate_bar.jpg")
+    bootstrap_elo_rating_file = args.clean_battle_file.replace(".json", "_bootstrap_elo_rating.jpg")
     anony_results["win_fraction_heatmap"].write_image(win_fraction_heatmap_file)
-    # anony_results["battle_count_heatmap"].write_image(battle_count_heatmap_file)
-    # anony_results["average_win_rate_bar"].write_image(average_win_rate_bar_file)
-    # anony_results["bootstrap_elo_rating"].write_image(bootstrap_elo_rating_file)
+    anony_results["battle_count_heatmap"].write_image(battle_count_heatmap_file)
+    anony_results["average_win_rate_bar"].write_image(average_win_rate_bar_file)
+    anony_results["bootstrap_elo_rating"].write_image(bootstrap_elo_rating_file)
     
 
     last_updated_tstamp = full_results["last_updated_tstamp"]

arena_elo/results/20240602/elo_results_image_editing.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51cebad92fed9a2c15871c3dae4ef1ffbbaea02b5906f3100e5e9f1dc25e97da
-size 62732
+oid sha256:41c7f35c5812940344d1b3f626339820480de7a40b029df9a1ce62f969833173
+size 62454

arena_elo/results/20240604/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:787326f9d085b668bdf7d2f24439f7080a402c7333423f4856c86ef1c11288b8
-size 68373
+oid sha256:7be5eb1f9434a7c8dcf82d1a530c1d4dc4bddfd6e9c1bec3c3bb215e50730a1d
+size 68116

arena_elo/results/20240604/elo_results_video_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e67b05d84d6bccdaefaba734220e3a87d1ef897a4969f415a8a94831a2a9d9a1
-size 60162
+oid sha256:b481ad17c28dd03dc7acd1d8236ba71b175cbde4b56d06dd926f35199cfb3831
+size 59905

arena_elo/results/20240605/elo_results_image_editing.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d399036e9c8820f0de8d75a502d79664fb39208b4f1313fbd7904444776891a
+size 62454

arena_elo/results/20240605/elo_results_t2i_generation.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:087fe327dbd4ab65eb4bbea49a98654b54e71fa0ad653f54190cea7bcc917ed6
+size 68115

arena_elo/results/20240605/elo_results_video_generation.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc59a521a30af39418ca44a2e8abe210a24c091a817e319ac9a19865819330fe
+size 59905

arena_elo/results/20240605/image_editing_leaderboard.csv

@@ -0,0 +1,10 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+MagicBrush,MagicBrush,1111.287883287791,1115.4920223628246,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
+InfEdit,InfEdit,1079.0215496678925,1078.7425836837601,CC BY-NC-ND 4.0,"University of Michigan, University of California, Berkeley",https://huggingface.co/spaces/sled-umich/InfEdit
+CosXLEdit,CosXLEdit,1065.6805346660533,1066.6576811167185,cosxl-nc-community,Stability AI,https://huggingface.co/spaces/multimodalart/cosxl
+InstructPix2Pix,InstructPix2Pix,1032.7530333195389,1030.5892525970432,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
+PNP,PNP,998.4983696664473,1003.328498120622,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
+Prompt2prompt,Prompt2prompt,987.9337296502758,989.1241997635233,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
+CycleDiffusion,CycleDiffusion,938.9517299322257,932.9343115136126,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
+SDEdit,SDEdit,928.5595445205557,926.8259267403399,MIT License,Stanford University,https://sde-image-editing.github.io
+Pix2PixZero,Pix2PixZero,857.3136252892202,856.3055241015554,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io

arena_elo/results/20240605/t2i_generation_leaderboard.csv

@@ -0,0 +1,12 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+PlayGround V2.5,PlayGround V2.5,1151.4765748556717,1153.6593332791817,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
+PlayGround V2,PlayGround V2,1100.3037040843228,1099.2629295943668,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+StableCascade,StableCascade,1057.3087770064067,1061.7664219410574,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+SDXLLightning,SDXLLightning,1052.18389910394,1056.4972274621996,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1051.5563906642553,1039.507593874199,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+PixArtSigma,PixArtSigma,1049.8702445682718,1049.1941177240374,openrail++,PixArt-alpha,https://fal.ai/models/fal-ai/pixart-sigma
+SDXL,SDXL,1000.4056267411033,998.804982828749,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,934.7707608687357,932.5634380637507,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+LCM(v1.5/XL),LCM(v1.5/XL),932.2617211496097,925.5957004497745,openrail++,Latent Consistency,https://fal.ai/models/fal-ai/fast-lcm-diffusion/api
+OpenJourney,OpenJourney,853.0015060982882,847.4884318387889,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,816.8607948593981,829.9259097682236,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240605/video_generation_leaderboard.csv

@@ -0,0 +1,9 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+T2VTurbo,T2VTurbo,1114.0976950076051,1111.996839823075,cc-by-nc-4.0,"University of California, Santa Barbara",https://huggingface.co/jiachenli-ucsb/T2V-Turbo-VC2
+StableVideoDiffusion,StableVideoDiffusion,1111.5916401252923,1115.2642690156,stable-video-diffusion-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
+VideoCrafter2,VideoCrafter2,1077.0075945696226,1077.7266263652966,Apache 2.0,Tencent AI Lab,https://ailab-cvc.github.io/videocrafter2/
+AnimateDiff,AnimateDiff,1073.1580940041636,1071.1675217333839,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v
+LaVie,LaVie,995.5394165448754,995.6499275888945,Apache 2.0,Shanghai AI Lab,https://github.com/Vchitect/LaVie
+OpenSora,OpenSora,914.4387176103126,914.0080243221488,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
+ModelScope,ModelScope,864.3033408750659,863.9078826243627,cc-by-nc-4.0,Alibaba Group,https://arxiv.org/abs/2308.06571
+AnimateDiffTurbo,AnimateDiffTurbo,849.8635012630624,850.2789085272368,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo

arena_elo/results/latest/elo_results_image_editing.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51cebad92fed9a2c15871c3dae4ef1ffbbaea02b5906f3100e5e9f1dc25e97da
-size 62732
+oid sha256:41c7f35c5812940344d1b3f626339820480de7a40b029df9a1ce62f969833173
+size 62454

arena_elo/results/latest/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:787326f9d085b668bdf7d2f24439f7080a402c7333423f4856c86ef1c11288b8
-size 68373
+oid sha256:7be5eb1f9434a7c8dcf82d1a530c1d4dc4bddfd6e9c1bec3c3bb215e50730a1d
+size 68116

arena_elo/results/latest/elo_results_video_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e67b05d84d6bccdaefaba734220e3a87d1ef897a4969f415a8a94831a2a9d9a1
-size 60162
+oid sha256:b481ad17c28dd03dc7acd1d8236ba71b175cbde4b56d06dd926f35199cfb3831
+size 59905

serve/gradio_web.py

@@ -16,7 +16,7 @@ from .vote_utils import (
     generate_igm_annoy_museum,
     share_js
 )
-from functools import partial
+from functools import partial 
         
 def build_side_by_side_ui_anony(models):
     notice_markdown = """
@@ -28,6 +28,11 @@ def build_side_by_side_ui_anony(models):
 - Vote won't be counted if model identity is revealed during conversation.
 - Click "New Round" to start a new round.
 
+## ⚠️ Data Collection Consent
+- Your input and votes will be collected for research purposes only. 
+- By using this service, you agree to the collection of your input and votes for research purposes. 
+- Your data will be anonymized and will not be used for commercial purposes.
+
 ## 🏆 Arena Elo 
 Find out who is the 🥇conditional image generation models! More models are going to be supported.
 
@@ -196,6 +201,11 @@ def build_side_by_side_ui_named(models):
 - Click "Send" to submit the prompt.
 - Click "Clear history" to start a new round.
 
+## ⚠️ Data Collection Consent
+- Your input and votes will be collected for research purposes only. 
+- By using this service, you agree to the collection of your input and votes for research purposes. 
+- Your data will be anonymized and will not be used for commercial purposes.
+
 ## 🤖 Choose two models to compare
 """
     model_list = models.model_ig_list

serve/gradio_web_image_editing.py

@@ -32,6 +32,11 @@ def build_side_by_side_ui_anony_ie(models):
 - Vote won't be counted if model identity is revealed during generation.
 - The model coulmodel_selector_left.changed output a totally black image or noise. It's not a bug but the failure case of the model.
 
+## ⚠️ Data Collection Consent
+- Your input and votes will be collected for research purposes only. 
+- By using this service, you agree to the collection of your input and votes for research purposes. 
+- Your data will be anonymized and will not be used for commercial purposes.
+
 ## 🏆 Arena Elo 
 Find out who is the 🥇conditional image edition models!
 
@@ -234,6 +239,11 @@ def build_side_by_side_ui_named_ie(models):
 - Click "Clear history" to start a new round.
 - The model could output a totally black image or noise. It's not a bug but the failure case of the model.
 
+## ⚠️ Data Collection Consent
+- Your input and votes will be collected for research purposes only. 
+- By using this service, you agree to the collection of your input and votes for research purposes. 
+- Your data will be anonymized and will not be used for commercial purposes.
+
 ## 🤖 Choose two models to compare
 """
 

serve/gradio_web_video_generation.py

@@ -29,6 +29,11 @@ def build_side_by_side_ui_anony_vg(models):
 - Vote won't be counted if model identity is revealed during conversation.
 - Click "New Round" to start a new round.
 
+## ⚠️ Data Collection Consent
+- Your input and votes will be collected for research purposes only. 
+- By using this service, you agree to the collection of your input and votes for research purposes. 
+- Your data will be anonymized and will not be used for commercial purposes.
+
 ## 🏆 Arena Elo 
 Find out who is the 🥇conditional video generation models! More models are going to be supported.
 
@@ -200,6 +205,11 @@ def build_side_by_side_ui_named_vg(models):
 - Click "Send" to submit the prompt.
 - Click "Clear history" to start a new round.
 
+## ⚠️ Data Collection Consent
+- Your input and votes will be collected for research purposes only. 
+- By using this service, you agree to the collection of your input and votes for research purposes. 
+- Your data will be anonymized and will not be used for commercial purposes.
+
 ## 🤖 Choose two models to compare
 """
     model_list = models.model_vg_list