update

arena_elo/elo_rating/clean_battle_data.py

@@ -79,6 +79,7 @@ def to_openai_format(messages):
 
 
 def replace_model_name(old_name, tstamp):
+    
     replace_dict = {
         "bard": "palm-2",
         "claude-v1": "claude-1",
@@ -87,14 +88,19 @@ def replace_model_name(old_name, tstamp):
         "claude-2": "claude-2.0",
         "PlayGroundV2": "PlayGround V2",
         "PlayGroundV2.5": "PlayGround V2.5",
+        "FluxTimestep": "FLUX1schnell",
+        "FluxGuidance": "FLUX1dev"
     }
     if old_name in ["gpt-4", "gpt-3.5-turbo"]:
         if tstamp > 1687849200:
-            return old_name + "-0613"
+            old_name += "-0613"
         else:
-            return old_name + "-0314"
+            old_name += "-0314"
     if old_name in replace_dict:
-        return replace_dict[old_name]
+        old_name =  replace_dict[old_name]
+    if "Flux" in old_name:
+        print(f"Invalid model names: {old_name}")
+        exit(1)
     return old_name
 
 
@@ -305,6 +311,10 @@ def clean_battle_data(
 
         else:
             raise ValueError(f"Invalid task_name: {task_name}")
+        
+        # if "Flux" in models[0] or "Flux" in models[1]:
+        #     print(f"Invalid model names: {models}")
+        #     exit(1)
         models = [replace_model_name(m, row["tstamp"]) for m in models]
         
         # Exclude certain models

arena_elo/elo_rating/elo_analysis.py

@@ -452,3 +452,6 @@ if __name__ == "__main__":
     }
     with open(f"elo_results_{cutoff_date}.pkl", "wb") as fout:
         pickle.dump(results, fout)
+
+    with open("cut_off_date.txt", "w") as fout:
+        fout.write(cutoff_date)

arena_elo/results/20240809/elo_results_image_editing.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1da1e19486e987e18cf9bec79c36f1b71291a4f92490cf7d3af4edbc98628510
-size 62482
+oid sha256:b54d460d93f22ee6e2519ccc82e4ebd64ec81987505760c428b593ff91a13368
+size 62455

arena_elo/results/20240809/image_editing_leaderboard.csv

@@ -2,7 +2,7 @@ key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,
 MagicBrush,MagicBrush,1112.4518849527376,1116.3228088792275,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
 InfEdit,InfEdit,1077.2249767545911,1077.0861206067664,CC BY-NC-ND 4.0,"University of Michigan, University of California, Berkeley",https://huggingface.co/spaces/sled-umich/InfEdit
 CosXLEdit,CosXLEdit,1064.682441262532,1065.599445502775,cosxl-nc-community,Stability AI,https://huggingface.co/spaces/multimodalart/cosxl
-InstructPix2Pix,InstructPix2Pix,1035.4054226880567,1033.208668133612,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
+InstructPix2Pix,InstructPix2Pix,1035.4054226880567,1033.2086681336118,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
 PNP,PNP,1001.4916130267786,1006.0747389504991,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
 Prompt2prompt,Prompt2prompt,990.9755779591587,992.0796321838297,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
 CycleDiffusion,CycleDiffusion,938.5637450408492,931.6722574678079,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion

arena_elo/{elo_results_20240809.pkl → results/20240818/elo_results_t2i_generation.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b54d460d93f22ee6e2519ccc82e4ebd64ec81987505760c428b593ff91a13368
-size 62455
+oid sha256:fa69fb8e450bd988c80b346d4f102c5be8bd74605030f0d5a295a71b66230ccd
+size 76790

arena_elo/results/20240818/elo_results_video_generation.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c76c0dad67cabfdbda5beb2056bfaf9f669f7a5a3857716212f6f831516d5f9
+size 59905

arena_elo/results/20240818/t2i_generation_leaderboard.csv

@@ -0,0 +1,15 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+PlayGround V2.5,PlayGround V2.5,1141.5188909673816,1141.7574769353046,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
+FLUX1schnell,FLUX1schnell,1107.0181424085022,1112.4579783184781,N/A,N/A,N/A
+PlayGround V2,PlayGround V2,1083.848194525069,1081.1558760457797,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+HunyuanDiT,HunyuanDiT,1061.8376119998818,1045.8437711625322,tencent-hunyuan-community,Tencent,https://huggingface.co/Tencent-Hunyuan/HunyuanDiT
+StableCascade,StableCascade,1048.1457195754278,1050.1691240248065,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+SDXLLightning,SDXLLightning,1034.30472320188,1037.2932871766393,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1033.912116989071,1022.557442237083,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+PixArtSigma,PixArtSigma,1029.1408183981687,1027.4953277391191,openrail++,PixArt-alpha,https://fal.ai/models/fal-ai/pixart-sigma
+SD3,SD3,1006.8458978216129,1000.2491511468914,stabilityai-nc-research-community,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-3-medium
+SDXL,SDXL,975.3595050901268,974.4701361426086,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,922.5038319935961,918.6139210049913,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+LCM(v1.5/XL),LCM(v1.5/XL),916.2179215619182,908.6161501509581,openrail++,Latent Consistency,https://fal.ai/models/fal-ai/fast-lcm-diffusion/api
+OpenJourney,OpenJourney,838.9432903688682,832.1138397704597,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,800.403335098498,812.6668778296877,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/20240818/video_generation_leaderboard.csv

@@ -0,0 +1,9 @@
+key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
+StableVideoDiffusion,StableVideoDiffusion,1133.1838006160342,1135.648326348363,stable-video-diffusion-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
+T2VTurbo,T2VTurbo,1097.0055576746822,1095.4539969650828,cc-by-nc-4.0,"University of California, Santa Barbara",https://huggingface.co/jiachenli-ucsb/T2V-Turbo-VC2
+VideoCrafter2,VideoCrafter2,1074.299413538351,1074.604913704468,Apache 2.0,Tencent AI Lab,https://ailab-cvc.github.io/videocrafter2/
+AnimateDiff,AnimateDiff,1068.2089967499214,1066.8805974519646,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v
+LaVie,LaVie,995.82479731396,994.8703618669116,Apache 2.0,Shanghai AI Lab,https://github.com/Vchitect/LaVie
+OpenSora,OpenSora,913.3181800582577,913.0542295067128,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
+ModelScope,ModelScope,861.1074201988872,862.3583669429345,cc-by-nc-4.0,Alibaba Group,https://arxiv.org/abs/2308.06571
+AnimateDiffTurbo,AnimateDiffTurbo,857.0518338499064,857.1292072135634,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo

arena_elo/results/latest/clean_battle_t2i_generation.json

@@ -49848,8 +49848,8 @@
     "tstamp": 1723054634.5468
   },
   {
-    "model_a": "FluxGuidance",
-    "model_b": "FluxTimestep",
+    "model_a": "FLUX1dev",
+    "model_b": "FLUX1schnell",
     "winner": "model_b",
     "judge": "arena_user_10.16.12.147",
     "anony": true,
@@ -49880,8 +49880,8 @@
     "tstamp": 1723063074.7642
   },
   {
-    "model_a": "FluxTimestep",
-    "model_b": "FluxGuidance",
+    "model_a": "FLUX1schnell",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.14.231",
     "anony": true,
@@ -49897,14 +49897,14 @@
   },
   {
     "model_a": "AuraFlow",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.31.28",
     "anony": false,
     "tstamp": 1723067240.2924
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLLightning",
     "winner": "model_a",
     "judge": "arena_user_10.16.43.145",
@@ -49920,16 +49920,16 @@
     "tstamp": 1723078808.7741
   },
   {
-    "model_a": "FluxTimestep",
-    "model_b": "FluxGuidance",
+    "model_a": "FLUX1schnell",
+    "model_b": "FLUX1dev",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.31.28",
     "anony": false,
     "tstamp": 1723080943.3647
   },
   {
-    "model_a": "FluxTimestep",
-    "model_b": "FluxGuidance",
+    "model_a": "FLUX1schnell",
+    "model_b": "FLUX1dev",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.12.147",
     "anony": false,
@@ -50016,7 +50016,7 @@
     "tstamp": 1723160130.8681
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLLightning",
     "winner": "model_a",
     "judge": "arena_user_10.16.17.24",
@@ -50057,7 +50057,7 @@
   },
   {
     "model_a": "PixArtAlpha",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "tie",
     "judge": "arena_user_10.16.43.145",
     "anony": true,
@@ -50080,7 +50080,7 @@
     "tstamp": 1723160326.9647
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "PlayGround V2.5",
     "winner": "model_a",
     "judge": "arena_user_10.16.22.62",
@@ -50088,8 +50088,8 @@
     "tstamp": 1723165134.1363
   },
   {
-    "model_a": "FluxGuidance",
-    "model_b": "FluxTimestep",
+    "model_a": "FLUX1dev",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.22.62",
     "anony": true,
@@ -50105,7 +50105,7 @@
   },
   {
     "model_a": "OpenJourney",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.43.145",
     "anony": true,
@@ -50128,7 +50128,7 @@
     "tstamp": 1723186558.1923
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "StableCascade",
     "winner": "tie",
     "judge": "arena_user_10.16.43.145",
@@ -50144,7 +50144,7 @@
     "tstamp": 1723186599.4345
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SD3",
     "winner": "model_a",
     "judge": "arena_user_10.16.22.62",
@@ -50169,14 +50169,14 @@
   },
   {
     "model_a": "AuraFlow",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_b",
     "judge": "arena_user_10.16.43.145",
     "anony": true,
     "tstamp": 1723203750.3175
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "Kolors",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.22.62",
@@ -50208,7 +50208,7 @@
     "tstamp": 1723206395.3717
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "StableCascade",
     "winner": "model_a",
     "judge": "arena_user_10.16.12.147",
@@ -50225,14 +50225,14 @@
   },
   {
     "model_a": "PixArtSigma",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.43.250",
     "anony": true,
     "tstamp": 1723206430.7406
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLTurbo",
     "winner": "model_a",
     "judge": "arena_user_10.16.43.145",
@@ -50248,16 +50248,16 @@
     "tstamp": 1723224658.8908
   },
   {
-    "model_a": "FluxGuidance",
-    "model_b": "FluxTimestep",
+    "model_a": "FLUX1dev",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.36.191",
     "anony": false,
     "tstamp": 1723232348.385
   },
   {
-    "model_a": "FluxTimestep",
-    "model_b": "FluxGuidance",
+    "model_a": "FLUX1schnell",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.36.191",
     "anony": false,
@@ -50289,7 +50289,7 @@
   },
   {
     "model_a": "OpenJourney",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "tie",
     "judge": "arena_user_10.16.22.62",
     "anony": true,
@@ -50392,7 +50392,7 @@
     "tstamp": 1723338245.5496
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLTurbo",
     "winner": "model_a",
     "judge": "arena_user_10.16.43.145",
@@ -50416,7 +50416,7 @@
     "tstamp": 1723338272.1677
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "SD3",
     "winner": "model_a",
     "judge": "arena_user_10.16.17.24",
@@ -50513,7 +50513,7 @@
   },
   {
     "model_a": "SDXL",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_b",
     "judge": "arena_user_10.16.12.147",
     "anony": false,
@@ -50576,7 +50576,7 @@
     "tstamp": 1723416140.7059
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "HunyuanDiT",
     "winner": "model_a",
     "judge": "arena_user_10.16.22.62",
@@ -50601,7 +50601,7 @@
   },
   {
     "model_a": "HunyuanDiT",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.36.191",
     "anony": true,
@@ -50617,14 +50617,14 @@
   },
   {
     "model_a": "AuraFlow",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.12.147",
     "anony": true,
     "tstamp": 1723466162.8794
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "OpenJourney",
     "winner": "model_a",
     "judge": "arena_user_10.16.45.228",
@@ -50640,8 +50640,8 @@
     "tstamp": 1723481690.3525
   },
   {
-    "model_a": "FluxGuidance",
-    "model_b": "FluxTimestep",
+    "model_a": "FLUX1dev",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.45.228",
     "anony": true,
@@ -50656,7 +50656,7 @@
     "tstamp": 1723510064.126
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "SDXLTurbo",
     "winner": "model_b",
     "judge": "arena_user_10.16.17.24",
@@ -50665,7 +50665,7 @@
   },
   {
     "model_a": "PixArtAlpha",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.22.62",
     "anony": true,
@@ -50697,7 +50697,7 @@
   },
   {
     "model_a": "HunyuanDiT",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.22.62",
     "anony": true,
@@ -50768,7 +50768,7 @@
     "tstamp": 1723521805.5654
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "StableCascade",
     "winner": "model_a",
     "judge": "arena_user_10.16.12.147",
@@ -50777,7 +50777,7 @@
   },
   {
     "model_a": "SDXL",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.36.191",
     "anony": true,
@@ -50785,7 +50785,7 @@
   },
   {
     "model_a": "AuraFlow",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.45.228",
     "anony": true,
@@ -50848,7 +50848,7 @@
     "tstamp": 1723526723.0377
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "PlayGround V2.5",
     "winner": "model_a",
     "judge": "arena_user_10.16.17.24",
@@ -50872,7 +50872,7 @@
     "tstamp": 1723568683.2572
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "PlayGround V2.5",
     "winner": "model_b",
     "judge": "arena_user_10.16.18.162",
@@ -50897,7 +50897,7 @@
   },
   {
     "model_a": "Kolors",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "tie",
     "judge": "arena_user_10.16.2.60",
     "anony": true,
@@ -50928,7 +50928,7 @@
     "tstamp": 1723644206.2483
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXL",
     "winner": "model_a",
     "judge": "arena_user_10.16.3.140",
@@ -51009,7 +51009,7 @@
   },
   {
     "model_a": "SD3",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "tie",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -51041,14 +51041,14 @@
   },
   {
     "model_a": "SDXLLightning",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
     "tstamp": 1723694936.0013
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLLightning",
     "winner": "model_b",
     "judge": "arena_user_10.16.18.75",
@@ -51080,7 +51080,7 @@
     "tstamp": 1723764843.3515
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLTurbo",
     "winner": "model_b",
     "judge": "arena_user_10.16.18.75",
@@ -51088,7 +51088,7 @@
     "tstamp": 1723764855.5548
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "PixArtSigma",
     "winner": "model_b",
     "judge": "arena_user_10.16.1.165",
@@ -51104,7 +51104,7 @@
     "tstamp": 1723764889.9278
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SD3",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.18.75",
@@ -51129,7 +51129,7 @@
   },
   {
     "model_a": "Kolors",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "tie",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
@@ -51161,7 +51161,7 @@
   },
   {
     "model_a": "PixArtAlpha",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -51192,7 +51192,7 @@
     "tstamp": 1723787634.5486
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXL",
     "winner": "model_a",
     "judge": "arena_user_10.16.1.165",
@@ -51209,7 +51209,7 @@
   },
   {
     "model_a": "SDXL",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_b",
     "judge": "arena_user_10.16.34.60",
     "anony": true,
@@ -51224,7 +51224,7 @@
     "tstamp": 1723814629.9803
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLLightning",
     "winner": "model_a",
     "judge": "arena_user_10.16.34.60",
@@ -51240,7 +51240,7 @@
     "tstamp": 1723824350.2469
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "AuraFlow",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.75",
@@ -51248,7 +51248,7 @@
     "tstamp": 1723824360.0778
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "PixArtSigma",
     "winner": "model_a",
     "judge": "arena_user_10.16.34.60",
@@ -51288,7 +51288,7 @@
     "tstamp": 1723838955.3114
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "OpenJourney",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.75",
@@ -51368,7 +51368,7 @@
     "tstamp": 1723839891.4319
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXLLightning",
     "winner": "model_b",
     "judge": "arena_user_10.16.34.60",
@@ -51376,7 +51376,7 @@
     "tstamp": 1723839938.2449
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SDXL",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.162",
@@ -51401,15 +51401,15 @@
   },
   {
     "model_a": "AuraFlow",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.34.60",
     "anony": true,
     "tstamp": 1723840344.8034
   },
   {
-    "model_a": "FluxTimestep",
-    "model_b": "FluxGuidance",
+    "model_a": "FLUX1schnell",
+    "model_b": "FLUX1dev",
     "winner": "tie",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
@@ -51417,7 +51417,7 @@
   },
   {
     "model_a": "AuraFlow",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_a",
     "judge": "arena_user_10.16.34.60",
     "anony": true,
@@ -51441,7 +51441,7 @@
   },
   {
     "model_a": "Kolors",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
@@ -51480,7 +51480,7 @@
     "tstamp": 1723855363.9435
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "SDXLTurbo",
     "winner": "tie",
     "judge": "arena_user_10.16.34.60",
@@ -51536,8 +51536,8 @@
     "tstamp": 1723855411.5431
   },
   {
-    "model_a": "FluxGuidance",
-    "model_b": "FluxTimestep",
+    "model_a": "FLUX1dev",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -51577,7 +51577,7 @@
   },
   {
     "model_a": "AuraFlow",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_b",
     "judge": "arena_user_10.16.34.60",
     "anony": true,
@@ -51585,7 +51585,7 @@
   },
   {
     "model_a": "Kolors",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "tie",
     "judge": "arena_user_10.16.34.60",
     "anony": true,
@@ -51641,7 +51641,7 @@
   },
   {
     "model_a": "SDXLTurbo",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.34.60",
     "anony": true,
@@ -51656,7 +51656,7 @@
     "tstamp": 1723855483.6681
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "OpenJourney",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.162",
@@ -51665,7 +51665,7 @@
   },
   {
     "model_a": "SDXL",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -51712,7 +51712,7 @@
     "tstamp": 1723855518.1443
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "AuraFlow",
     "winner": "tie",
     "judge": "arena_user_10.16.18.162",
@@ -51728,7 +51728,7 @@
     "tstamp": 1723855533.5625
   },
   {
-    "model_a": "FluxGuidance",
+    "model_a": "FLUX1dev",
     "model_b": "SD3",
     "winner": "model_a",
     "judge": "arena_user_10.16.34.60",
@@ -51745,7 +51745,7 @@
   },
   {
     "model_a": "Kolors",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_b",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -51769,7 +51769,7 @@
   },
   {
     "model_a": "SDXLLightning",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "tie",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -51792,7 +51792,7 @@
     "tstamp": 1723855564.8518
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "PixArtSigma",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.75",
@@ -51833,7 +51833,7 @@
   },
   {
     "model_a": "PlayGround V2.5",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -51841,7 +51841,7 @@
   },
   {
     "model_a": "PixArtAlpha",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "model_b",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
@@ -51849,7 +51849,7 @@
   },
   {
     "model_a": "PlayGround V2.5",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_a",
     "judge": "arena_user_10.16.34.60",
     "anony": true,
@@ -51865,7 +51865,7 @@
   },
   {
     "model_a": "PlayGround V2.5",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "tie",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
@@ -51881,14 +51881,14 @@
   },
   {
     "model_a": "HunyuanDiT",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
     "tstamp": 1723855622.7164
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "SD3",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.18.75",
@@ -51913,7 +51913,7 @@
   },
   {
     "model_a": "Kolors",
-    "model_b": "FluxTimestep",
+    "model_b": "FLUX1schnell",
     "winner": "tie (bothbad)",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
@@ -52057,7 +52057,7 @@
   },
   {
     "model_a": "PlayGround V2.5",
-    "model_b": "FluxGuidance",
+    "model_b": "FLUX1dev",
     "winner": "model_a",
     "judge": "arena_user_10.16.18.162",
     "anony": true,
@@ -52072,7 +52072,7 @@
     "tstamp": 1723876446.6019
   },
   {
-    "model_a": "FluxTimestep",
+    "model_a": "FLUX1schnell",
     "model_b": "PlayGround V2.5",
     "winner": "model_b",
     "judge": "arena_user_10.16.18.162",
@@ -52168,11 +52168,243 @@
     "tstamp": 1723908029.3549
   },
   {
-    "model_a": "FluxGuidance",
-    "model_b": "FluxTimestep",
+    "model_a": "FLUX1dev",
+    "model_b": "FLUX1schnell",
     "winner": "tie",
     "judge": "arena_user_10.16.18.75",
     "anony": true,
     "tstamp": 1723908074.195
+  },
+  {
+    "model_a": "OpenJourney",
+    "model_b": "FLUX1dev",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723910785.0692
+  },
+  {
+    "model_a": "SDXLTurbo",
+    "model_b": "AuraFlow",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922566.9706
+  },
+  {
+    "model_a": "StableCascade",
+    "model_b": "SDXLTurbo",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_10.16.1.165",
+    "anony": true,
+    "tstamp": 1723922575.0171
+  },
+  {
+    "model_a": "PixArtSigma",
+    "model_b": "Kolors",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922584.553
+  },
+  {
+    "model_a": "SDXLTurbo",
+    "model_b": "StableCascade",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922593.6921
+  },
+  {
+    "model_a": "FLUX1schnell",
+    "model_b": "PixArtAlpha",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922618.7522
+  },
+  {
+    "model_a": "StableCascade",
+    "model_b": "PixArtSigma",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922626.6249
+  },
+  {
+    "model_a": "SD3",
+    "model_b": "SDXLTurbo",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922643.7963
+  },
+  {
+    "model_a": "Kolors",
+    "model_b": "SDXLLightning",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723922667.2898
+  },
+  {
+    "model_a": "FLUX1dev",
+    "model_b": "SDXLTurbo",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922745.1648
+  },
+  {
+    "model_a": "Kolors",
+    "model_b": "FLUX1schnell",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723922759.0787
+  },
+  {
+    "model_a": "PlayGround V2.5",
+    "model_b": "Kolors",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922767.444
+  },
+  {
+    "model_a": "OpenJourney",
+    "model_b": "PixArtSigma",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922776.0625
+  },
+  {
+    "model_a": "StableCascade",
+    "model_b": "SDXLTurbo",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922788.5956
+  },
+  {
+    "model_a": "SDXLTurbo",
+    "model_b": "SDXLLightning",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922800.3378
+  },
+  {
+    "model_a": "SDXLTurbo",
+    "model_b": "PixArtAlpha",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.1.165",
+    "anony": true,
+    "tstamp": 1723922808.3497
+  },
+  {
+    "model_a": "SD3",
+    "model_b": "PixArtSigma",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723922819.7194
+  },
+  {
+    "model_a": "SD3",
+    "model_b": "StableCascade",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723922828.2665
+  },
+  {
+    "model_a": "PixArtAlpha",
+    "model_b": "AuraFlow",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.1.165",
+    "anony": true,
+    "tstamp": 1723963692.1376
+  },
+  {
+    "model_a": "SDXLLightning",
+    "model_b": "OpenJourney",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723963701.4014
+  },
+  {
+    "model_a": "SDXLLightning",
+    "model_b": "FLUX1dev",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.1.165",
+    "anony": true,
+    "tstamp": 1723963715.181
+  },
+  {
+    "model_a": "SDXL",
+    "model_b": "SD3",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.18.162",
+    "anony": true,
+    "tstamp": 1723963739.8545
+  },
+  {
+    "model_a": "AuraFlow",
+    "model_b": "SDXL",
+    "winner": "model_b",
+    "judge": "arena_user_10.16.1.165",
+    "anony": true,
+    "tstamp": 1723963746.2871
+  },
+  {
+    "model_a": "HunyuanDiT",
+    "model_b": "SDXL",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.1.165",
+    "anony": true,
+    "tstamp": 1723970594.9832
+  },
+  {
+    "model_a": "StableCascade",
+    "model_b": "PlayGround V2.5",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723970603.8367
+  },
+  {
+    "model_a": "PixArtAlpha",
+    "model_b": "SDXL",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723970610.2448
+  },
+  {
+    "model_a": "PixArtSigma",
+    "model_b": "AuraFlow",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723970619.6508
+  },
+  {
+    "model_a": "SD3",
+    "model_b": "SDXLLightning",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723970626.3099
+  },
+  {
+    "model_a": "FLUX1schnell",
+    "model_b": "OpenJourney",
+    "winner": "model_a",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1723970632.9565
   }
 ]

arena_elo/results/latest/clean_battle_video_generation.json

@@ -14862,5 +14862,21 @@
     "judge": "arena_user_10.16.18.75",
     "anony": true,
     "tstamp": 1723909957.9225
+  },
+  {
+    "model_a": "AnimateDiff",
+    "model_b": "AnimateDiffTurbo",
+    "winner": "tie (bothbad)",
+    "judge": "arena_user_10.16.1.165",
+    "anony": true,
+    "tstamp": 1723963592.0484
+  },
+  {
+    "model_a": "AnimateDiff",
+    "model_b": "AnimateDiffTurbo",
+    "winner": "tie",
+    "judge": "arena_user_10.16.18.75",
+    "anony": true,
+    "tstamp": 1724006529.2433
   }
 ]

arena_elo/results/latest/elo_results_image_editing.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1da1e19486e987e18cf9bec79c36f1b71291a4f92490cf7d3af4edbc98628510
-size 62482
+oid sha256:b54d460d93f22ee6e2519ccc82e4ebd64ec81987505760c428b593ff91a13368
+size 62455

arena_elo/results/latest/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:503ad9814b462e2bd1e60a3bdfb27549a374706e5f622cf5848a776509b2bc0e
-size 76789
+oid sha256:fa69fb8e450bd988c80b346d4f102c5be8bd74605030f0d5a295a71b66230ccd
+size 76790

arena_elo/results/latest/elo_results_video_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3bfa72b72fe286920122d9d36e161dcab2a2ee06ecf5ceb6ad822fd50063548
+oid sha256:6c76c0dad67cabfdbda5beb2056bfaf9f669f7a5a3857716212f6f831516d5f9
 size 59905

arena_elo/results/latest/image_editing_leaderboard.csv

@@ -2,7 +2,7 @@ key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,
 MagicBrush,MagicBrush,1112.4518849527376,1116.3228088792275,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
 InfEdit,InfEdit,1077.2249767545911,1077.0861206067664,CC BY-NC-ND 4.0,"University of Michigan, University of California, Berkeley",https://huggingface.co/spaces/sled-umich/InfEdit
 CosXLEdit,CosXLEdit,1064.682441262532,1065.599445502775,cosxl-nc-community,Stability AI,https://huggingface.co/spaces/multimodalart/cosxl
-InstructPix2Pix,InstructPix2Pix,1035.4054226880567,1033.208668133612,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
+InstructPix2Pix,InstructPix2Pix,1035.4054226880567,1033.2086681336118,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
 PNP,PNP,1001.4916130267786,1006.0747389504991,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
 Prompt2prompt,Prompt2prompt,990.9755779591587,992.0796321838297,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
 CycleDiffusion,CycleDiffusion,938.5637450408492,931.6722574678079,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion

arena_elo/results/latest/t2i_generation_leaderboard.csv

@@ -1,15 +1,15 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGround V2.5,PlayGround V2.5,1141.866659999037,1141.8338384616795,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-FluxTimestep,FluxTimestep,1115.928646165151,1121.9304446567799,N/A,N/A,N/A
-PlayGround V2,PlayGround V2,1083.663321447569,1080.8791567769488,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-HunyuanDiT,HunyuanDiT,1059.7533365825295,1043.5339717960453,tencent-hunyuan-community,Tencent,https://huggingface.co/Tencent-Hunyuan/HunyuanDiT
-StableCascade,StableCascade,1046.7571473591233,1048.8389078449923,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-SDXLLightning,SDXLLightning,1034.8611228210834,1037.6419954765402,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1033.0085889123181,1021.5860196967267,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-PixArtSigma,PixArtSigma,1029.7824987127658,1027.9500586698966,openrail++,PixArt-alpha,https://fal.ai/models/fal-ai/pixart-sigma
-SD3,SD3,1001.3310047788816,994.5930116229433,stabilityai-nc-research-community,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-3-medium
-SDXL,SDXL,975.7678875209955,974.6391412818967,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,922.2975316059412,918.3190757057619,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-LCM(v1.5/XL),LCM(v1.5/XL),915.9954758760788,908.3365622923737,openrail++,Latent Consistency,https://fal.ai/models/fal-ai/fast-lcm-diffusion/api
-OpenJourney,OpenJourney,838.7888235391866,831.8551299360711,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,800.197954679335,812.3288511265314,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
+PlayGround V2.5,PlayGround V2.5,1141.5188909673816,1141.7574769353046,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
+FLUX1schnell,FLUX1schnell,1107.0181424085022,1112.4579783184781,N/A,N/A,N/A
+PlayGround V2,PlayGround V2,1083.848194525069,1081.1558760457797,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+HunyuanDiT,HunyuanDiT,1061.8376119998818,1045.8437711625322,tencent-hunyuan-community,Tencent,https://huggingface.co/Tencent-Hunyuan/HunyuanDiT
+StableCascade,StableCascade,1048.1457195754278,1050.1691240248065,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+SDXLLightning,SDXLLightning,1034.30472320188,1037.2932871766393,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1033.912116989071,1022.557442237083,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+PixArtSigma,PixArtSigma,1029.1408183981687,1027.4953277391191,openrail++,PixArt-alpha,https://fal.ai/models/fal-ai/pixart-sigma
+SD3,SD3,1006.8458978216129,1000.2491511468914,stabilityai-nc-research-community,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-3-medium
+SDXL,SDXL,975.3595050901268,974.4701361426086,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,922.5038319935961,918.6139210049913,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+LCM(v1.5/XL),LCM(v1.5/XL),916.2179215619182,908.6161501509581,openrail++,Latent Consistency,https://fal.ai/models/fal-ai/fast-lcm-diffusion/api
+OpenJourney,OpenJourney,838.9432903688682,832.1138397704597,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,800.403335098498,812.6668778296877,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/latest/video_generation_leaderboard.csv

@@ -1,9 +1,9 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-StableVideoDiffusion,StableVideoDiffusion,1133.2057266842914,1135.6740718576937,stable-video-diffusion-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
-T2VTurbo,T2VTurbo,1097.0362394075614,1095.481109681995,cc-by-nc-4.0,"University of California, Santa Barbara",https://huggingface.co/jiachenli-ucsb/T2V-Turbo-VC2
-VideoCrafter2,VideoCrafter2,1074.3273548011723,1074.6323977593943,Apache 2.0,Tencent AI Lab,https://ailab-cvc.github.io/videocrafter2/
-AnimateDiff,AnimateDiff,1068.93900863825,1067.5731621857178,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v
-LaVie,LaVie,995.8323112101649,994.8789456408321,Apache 2.0,Shanghai AI Lab,https://github.com/Vchitect/LaVie
-OpenSora,OpenSora,913.264882541298,913.001492764224,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
-ModelScope,ModelScope,861.0476363835537,862.3030529191846,cc-by-nc-4.0,Alibaba Group,https://arxiv.org/abs/2308.06571
-AnimateDiffTurbo,AnimateDiffTurbo,856.3468403337079,856.4557671909597,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo
+StableVideoDiffusion,StableVideoDiffusion,1133.1838006160342,1135.648326348363,stable-video-diffusion-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
+T2VTurbo,T2VTurbo,1097.0055576746822,1095.4539969650828,cc-by-nc-4.0,"University of California, Santa Barbara",https://huggingface.co/jiachenli-ucsb/T2V-Turbo-VC2
+VideoCrafter2,VideoCrafter2,1074.299413538351,1074.604913704468,Apache 2.0,Tencent AI Lab,https://ailab-cvc.github.io/videocrafter2/
+AnimateDiff,AnimateDiff,1068.2089967499214,1066.8805974519646,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v
+LaVie,LaVie,995.82479731396,994.8703618669116,Apache 2.0,Shanghai AI Lab,https://github.com/Vchitect/LaVie
+OpenSora,OpenSora,913.3181800582577,913.0542295067128,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
+ModelScope,ModelScope,861.1074201988872,862.3583669429345,cc-by-nc-4.0,Alibaba Group,https://arxiv.org/abs/2308.06571
+AnimateDiffTurbo,AnimateDiffTurbo,857.0518338499064,857.1292072135634,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo

arena_elo/update_elo_rating.sh

@@ -3,6 +3,7 @@ if [ -z "$LOGDIR" ]; then
     echo "LOGDIR is not set. Using default '../GenAI-Arena-hf-logs/vote_log'"
     export LOGDIR="../GenAI-Arena-hf-logs/vote_log"
 fi
+set -e
 
 mkdir -p results
 
@@ -29,13 +30,19 @@ mv clean_battle_t2i_generation_$generation_battle_cutoff_date.json ./results/$ge
 mv clean_battle_video_generation_$video_generation_battle_cutoff_date.json ./results/$video_generation_battle_cutoff_date/clean_battle_video_generation.json
 
 
+echo "Calculating Elo rating for image editing task"
 python3 -m elo_rating.elo_analysis --clean-battle-file ./results/$edition_battle_cutoff_date/clean_battle_image_editing.json
+edition_battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "Image editing battle data (after filtering models with < 50 battles) last updated on $edition_battle_cutoff_date"
 mv ./elo_results_$edition_battle_cutoff_date.pkl ./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl
 
+echo "Calculating Elo rating for t2i generation task"
 python3 -m elo_rating.elo_analysis --clean-battle-file ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
+generation_battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "T2I image generation battle data (after filtering models with < 50 battles) last updated on $generation_battle_cutoff_date"
 mv ./elo_results_$generation_battle_cutoff_date.pkl ./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl
 
+echo "Calculating Elo rating for video generation task"
 python3 -m elo_rating.elo_analysis --clean-battle-file ./results/$video_generation_battle_cutoff_date/clean_battle_video_generation.json
+video_generation_battle_cutoff_date=`cat cut_off_date.txt` && rm cut_off_date.txt && echo "Video generation battle data (after filtering models with < 50 battles) last updated on $video_generation_battle_cutoff_date"
 mv ./elo_results_$video_generation_battle_cutoff_date.pkl ./results/$video_generation_battle_cutoff_date/elo_results_video_generation.pkl
 
 # generat the leaderboard

model/model_manager.py

@@ -34,9 +34,10 @@ class ModelManager:
         model_id = "meta-llama/Llama-Guard-3-8B"
         device = "cuda" if torch.cuda.is_available() else "cpu"
         dtype = torch.bfloat16
+        token = os.getenv("HF_TOKEN") or os.getenv("HF_GUARD")
         if enable_nsfw:
-            self.guard_tokenizer = AutoTokenizer.from_pretrained(model_id, token=os.environ['HF_GUARD'])
-            self.guard = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=dtype, device_map=device, token=os.environ['HF_GUARD'])
+            self.guard_tokenizer = AutoTokenizer.from_pretrained(model_id)
+            self.guard = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=dtype, device_map=device)
         else:
             self.guard_tokenizer = None
             self.guard = None