update model registry

arena_elo/elo_rating/clean_battle_data.py

@@ -18,7 +18,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
 from tqdm import tqdm
 
 from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
-from .utils import detect_language, get_time_stamp_from_date
+from .utils import detect_language, get_time_stamp_from_date, get_model_info
 
 VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote"]
 
@@ -54,6 +54,8 @@ def replace_model_name(old_name, tstamp):
     if "Flux" in old_name:
         print(f"Invalid model names: {old_name}")
         exit(1)
+    model_info = get_model_info(old_name)
+    old_name = model_info.simple_name
     return old_name
 
 

arena_elo/elo_rating/elo_analysis.py

@@ -11,9 +11,9 @@ import pandas as pd
 import plotly.express as px
 from tqdm import tqdm
 
-from .model_registry import get_model_info
 from .basic_stats import get_log_files
 from .clean_battle_data import clean_battle_data
+from .utils import get_model_info
         
 pd.options.display.float_format = "{:.2f}".format
 

arena_elo/elo_rating/generate_leaderboard.py

@@ -2,15 +2,12 @@ import fire
 import json
 import pandas as pd
 import pickle
-
+from .utils import get_model_info
 
 def main(
-        model_info_file: str,
-        elo_rating_pkl: str,
-        output_csv: str
+    elo_rating_pkl: str,
+    output_csv: str
 ):
-    model_info = json.load(open(model_info_file))
-
     with open(elo_rating_pkl, "rb") as fin:
         elo_rating_results = pickle.load(fin)
 
@@ -19,20 +16,23 @@ def main(
     anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
     full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
 
+    print(anony_leaderboard_data)
     # Model,MT-bench (score),Arena Elo rating,MMLU,License,Link
     fields = ["key", "Model", "Arena Elo rating (anony)", "Arena Elo rating (full)", "License", "Organization", "Link"]
     # set Organization and license to empty for now
     all_models = anony_leaderboard_data.index.tolist()
 
+    model_info = {}
     for model in all_models:
-        if not model in model_info:
-            model_info[model] = {}
-            model_info[model]["License"] = "N/A"
-            model_info[model]["Organization"] = "N/A"
-            model_info[model]["Link"] = "N/A"
-            print(f"Model {model} not found in model_info.json")
-        model_info[model]["Model"] = model
-        model_info[model]["key"] = model
+    
+        registered_model_info = get_model_info(model)
+        model_info[model] = {
+            "key": model,
+            "Model": model,
+            "License": registered_model_info.license,
+            "Organization": registered_model_info.organization,
+            "Link": registered_model_info.link
+        }
 
         if model in anony_leaderboard_data.index:
             model_info[model]["Arena Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]

arena_elo/elo_rating/model_registry.py

@@ -1,578 +0,0 @@
-"""Additional information of the models."""
-from collections import namedtuple, OrderedDict
-from typing import List
-
-
-ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description"])
-
-
-model_info = OrderedDict()
-
-
-def register_model_info(
-    full_names: List[str], simple_name: str, link: str, description: str
-):
-    info = ModelInfo(simple_name, link, description)
-
-    for full_name in full_names:
-        model_info[full_name] = info
-
-
-def get_model_info(name: str) -> ModelInfo:
-    if name in model_info:
-        return model_info[name]
-    else:
-        # To fix this, please use `register_model_info` to register your model
-        return ModelInfo(
-            name, "", "Register the description at arena.model/model_registry.py"
-        )
-
-
-register_model_info(
-    [
-        "IEITYuan/Yuan2-2B-Janus-hf",
-        "IEITYuan/Yuan2-2B-hf",
-        "IEITYuan/Yuan2-51B-hf",
-        "IEITYuan/Yuan2-102B-hf",
-    ],
-    "IEIT-Yuan2",
-    "https://github.com/IEIT-Yuan/Yuan-2.0",
-    "Yuan2.0 is a new generation Fundamental Large Language Model developed by IEIT System.",
-)
-
-register_model_info(
-    ["mixtral-8x7b-instruct-v0.1", "mistral-7b-instruct"],
-    "Mixtral of experts",
-    "https://mistral.ai/news/mixtral-of-experts/",
-    "A Mixture-of-Experts model by Mistral AI",
-)
-
-register_model_info(
-    ["gemini-pro"],
-    "Gemini",
-    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
-    "Gemini by Google",
-)
-
-register_model_info(
-    ["gemini-pro-vision"],
-    "Gemini",
-    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
-    "Gemini by Google",
-)
-
-register_model_info(
-    ["solar-10.7b-instruct-v1.0"],
-    "SOLAR-10.7B-Instruct",
-    "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0",
-    "A model trained using depth up-scaling by Upstage AI",
-)
-
-register_model_info(
-    ["gpt-4-turbo"],
-    "GPT-4-Turbo",
-    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
-    "GPT-4-Turbo by OpenAI",
-)
-
-register_model_info(
-    ["gpt-4-vision-preview"],
-    "gpt-4-vision-preview",
-    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
-    "GPT-4(V) by OpenAI",
-)
-
-register_model_info(
-    ["gpt-3.5-turbo", "gpt-3.5-turbo-0314", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-1106"],
-    "GPT-3.5",
-    "https://platform.openai.com/docs/models/gpt-3-5",
-    "GPT-3.5-Turbo by OpenAI",
-)
-
-register_model_info(
-    ["gpt-4", "gpt-4-0314", "gpt-4-0613"],
-    "GPT-4",
-    "https://openai.com/research/gpt-4",
-    "GPT-4 by OpenAI",
-)
-
-register_model_info(
-    ["claude-2.1", "claude-2.0"],
-    "Claude",
-    "https://www.anthropic.com/index/claude-2",
-    "Claude 2 by Anthropic",
-)
-
-register_model_info(
-    ["claude-1"],
-    "Claude",
-    "https://www.anthropic.com/index/introducing-claude",
-    "Claude 1 by Anthropic",
-)
-
-register_model_info(
-    ["claude-instant-1", "claude-instant-1.2"],
-    "Claude Instant",
-    "https://www.anthropic.com/index/introducing-claude",
-    "Claude Instant by Anthropic",
-)
-
-register_model_info(
-    ["pplx-70b-online", "pplx-7b-online"],
-    "pplx-online-llms",
-    "https://blog.perplexity.ai/blog/introducing-pplx-online-llms",
-    "Online LLM API by Perplexity AI",
-)
-
-register_model_info(
-    ["openhermes-2.5-mistral-7b"],
-    "OpenHermes-2.5-Mistral-7B",
-    "https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B",
-    "a mistral-based model fine-tuned on 1M GPT-4 outputs",
-)
-
-register_model_info(
-    ["starling-lm-7b-alpha"],
-    "Starling-LM-7B-alpha",
-    "https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha",
-    "an open model trained using RLAIF by Berkeley",
-)
-
-register_model_info(
-    ["tulu-2-dpo-70b"],
-    "Tulu 2",
-    "https://huggingface.co/allenai/tulu-2-dpo-70b",
-    "an instruction and RLHF model by UW/AllenAI",
-)
-
-register_model_info(
-    ["yi-34b-chat", "yi-6b-chat"],
-    "Yi-Chat",
-    "https://huggingface.co/01-ai/Yi-34B-Chat",
-    "A large language model by 01 AI",
-)
-
-register_model_info(
-    ["llama-2-70b-chat", "llama-2-34b-chat", "llama-2-13b-chat", "llama-2-7b-chat"],
-    "Llama 2",
-    "https://ai.meta.com/llama/",
-    "open foundation and fine-tuned chat models by Meta",
-)
-
-register_model_info(
-    [
-        "vicuna-33b",
-        "vicuna-33b-v1.3",
-        "vicuna-13b",
-        "vicuna-13b-v1.3",
-        "vicuna-7b",
-        "vicuna-7b-v1.3",
-    ],
-    "Vicuna",
-    "https://lmsys.org/blog/2023-03-30-vicuna/",
-    "a chat assistant fine-tuned on user-shared conversations by LMSYS",
-)
-
-register_model_info(
-    ["chatglm3-6b", "chatglm2-6b", "chatglm-6b"],
-    "ChatGLM",
-    "https://chatglm.cn/blog",
-    "an open bilingual dialogue language model by Tsinghua University",
-)
-
-register_model_info(
-    ["openchat-3.5"],
-    "OpenChat 3.5",
-    "https://github.com/imoneoi/openchat",
-    "an open model fine-tuned on Mistral-7B using C-RLFT",
-)
-
-register_model_info(
-    ["tenyxchat-7b-v1"],
-    "TenyxChat-7B",
-    "https://huggingface.co/tenyx/TenyxChat-7B-v1",
-    "an open model DPO trained on top of OpenChat-3.5 using Tenyx fine-tuning",
-)
-
-register_model_info(
-    ["zephyr-7b-beta", "zephyr-7b-alpha"],
-    "Zephyr",
-    "https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha",
-    "a chatbot fine-tuned from Mistral by Hugging Face",
-)
-
-register_model_info(
-    ["notus-7b-v1"],
-    "Notus",
-    "https://huggingface.co/argilla/notus-7b-v1",
-    "a chatbot fine-tuned from Zephyr SFT by Argilla",
-)
-
-register_model_info(
-    ["catppt"],
-    "CatPPT",
-    "https://huggingface.co/rishiraj/CatPPT",
-    "a chatbot fine-tuned from a SLERP merged model by Rishiraj Acharya",
-)
-
-register_model_info(
-    ["TinyLlama"],
-    "TinyLlama",
-    "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "The TinyLlama project is an open endeavor to pretrain a 1.1B Llama model on 3 trillion tokens.",
-)
-
-register_model_info(
-    ["qwen-14b-chat"],
-    "Qwen",
-    "https://huggingface.co/Qwen/Qwen-14B-Chat",
-    "a large language model by Alibaba Cloud",
-)
-
-register_model_info(
-    ["codellama-34b-instruct", "codellama-13b-instruct", "codellama-7b-instruct"],
-    "Code Llama",
-    "https://ai.meta.com/blog/code-llama-large-language-model-coding/",
-    "open foundation models for code by Meta",
-)
-
-register_model_info(
-    ["wizardlm-70b", "wizardlm-30b", "wizardlm-13b"],
-    "WizardLM",
-    "https://github.com/nlpxucan/WizardLM",
-    "an instruction-following LLM using evol-instruct by Microsoft",
-)
-
-register_model_info(
-    ["wizardcoder-15b-v1.0"],
-    "WizardLM",
-    "https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder",
-    "Empowering Code Large Language Models with Evol-Instruct",
-)
-
-register_model_info(
-    ["mpt-7b-chat", "mpt-30b-chat"],
-    "MPT-Chat",
-    "https://www.mosaicml.com/blog/mpt-30b",
-    "a chatbot fine-tuned from MPT by MosaicML",
-)
-
-register_model_info(
-    ["guanaco-33b", "guanaco-65b"],
-    "Guanaco",
-    "https://github.com/artidoro/qlora",
-    "a model fine-tuned with QLoRA by UW",
-)
-
-register_model_info(
-    ["gpt4all-13b-snoozy"],
-    "GPT4All-Snoozy",
-    "https://github.com/nomic-ai/gpt4all",
-    "a finetuned LLaMA model on assistant style data by Nomic AI",
-)
-
-register_model_info(
-    ["koala-13b"],
-    "Koala",
-    "https://bair.berkeley.edu/blog/2023/04/03/koala",
-    "a dialogue model for academic research by BAIR",
-)
-
-register_model_info(
-    ["RWKV-4-Raven-14B"],
-    "RWKV-4-Raven",
-    "https://huggingface.co/BlinkDL/rwkv-4-raven",
-    "an RNN with transformer-level LLM performance",
-)
-
-register_model_info(
-    ["alpaca-13b"],
-    "Alpaca",
-    "https://crfm.stanford.edu/2023/03/13/alpaca.html",
-    "a model fine-tuned from LLaMA on instruction-following demonstrations by Stanford",
-)
-
-register_model_info(
-    ["oasst-pythia-12b"],
-    "OpenAssistant (oasst)",
-    "https://open-assistant.io",
-    "an Open Assistant for everyone by LAION",
-)
-
-register_model_info(
-    ["oasst-sft-7-llama-30b"],
-    "OpenAssistant (oasst)",
-    "https://open-assistant.io",
-    "an Open Assistant for everyone by LAION",
-)
-
-register_model_info(
-    ["palm-2"],
-    "PaLM 2 Chat",
-    "https://cloud.google.com/vertex-ai/docs/release-notes#May_10_2023",
-    "PaLM 2 for Chat (chat-bison@001) by Google",
-)
-
-register_model_info(
-    ["llama-7b", "llama-13b"],
-    "LLaMA",
-    "https://arxiv.org/abs/2302.13971",
-    "open and efficient foundation language models by Meta",
-)
-
-register_model_info(
-    ["open-llama-7b-v2-open-instruct", "open-llama-7b-open-instruct"],
-    "Open LLaMa (Open Instruct)",
-    "https://medium.com/vmware-data-ml-blog/starter-llm-for-the-enterprise-instruction-tuning-openllama-7b-d05fc3bbaccc",
-    "Open LLaMa fine-tuned on instruction-following data by VMware",
-)
-
-register_model_info(
-    ["dolly-v2-12b"],
-    "Dolly",
-    "https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm",
-    "an instruction-tuned open large language model by Databricks",
-)
-
-register_model_info(
-    ["stablelm-tuned-alpha-7b"],
-    "StableLM",
-    "https://github.com/stability-AI/stableLM",
-    "Stability AI language models",
-)
-
-register_model_info(
-    ["codet5p-6b"],
-    "CodeT5p-6b",
-    "https://huggingface.co/Salesforce/codet5p-6b",
-    "Code completion model released by Salesforce",
-)
-
-register_model_info(
-    ["fastchat-t5-3b", "fastchat-t5-3b-v1.0"],
-    "FastChat-T5",
-    "https://huggingface.co/lmsys/fastchat-t5-3b-v1.0",
-    "a chat assistant fine-tuned from FLAN-T5 by LMSYS",
-)
-
-register_model_info(
-    ["phoenix-inst-chat-7b"],
-    "Phoenix-7B",
-    "https://huggingface.co/FreedomIntelligence/phoenix-inst-chat-7b",
-    "a multilingual chat assistant fine-tuned from Bloomz to democratize ChatGPT across languages by CUHK(SZ)",
-)
-
-register_model_info(
-    ["realm-7b-v1"],
-    "ReaLM",
-    "https://github.com/FreedomIntelligence/ReaLM",
-    "A chatbot fine-tuned from LLaMA2 with data generated via iterative calls to UserGPT and ChatGPT by CUHK(SZ) and SRIBD.",
-)
-
-register_model_info(
-    ["billa-7b-sft"],
-    "BiLLa-7B-SFT",
-    "https://huggingface.co/Neutralzz/BiLLa-7B-SFT",
-    "an instruction-tuned bilingual LLaMA with enhanced reasoning ability by an independent researcher",
-)
-
-register_model_info(
-    ["h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2"],
-    "h2oGPT-GM-7b",
-    "https://huggingface.co/h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2",
-    "an instruction-tuned OpenLLaMA with enhanced conversational ability by H2O.ai",
-)
-
-register_model_info(
-    ["baize-v2-7b", "baize-v2-13b"],
-    "Baize v2",
-    "https://github.com/project-baize/baize-chatbot#v2",
-    "A chatbot fine-tuned from LLaMA with ChatGPT self-chat data and Self-Disillation with Feedback (SDF) by UCSD and SYSU.",
-)
-
-register_model_info(
-    [
-        "airoboros-l2-7b-2.1",
-        "airoboros-l2-13b-2.1",
-        "airoboros-c34b-2.1",
-        "airoboros-l2-70b-2.1",
-    ],
-    "airoboros",
-    "https://huggingface.co/jondurbin/airoboros-l2-70b-2.1",
-    "an instruction-tuned LlaMa model tuned with 100% synthetic instruction-response pairs from GPT4",
-)
-
-register_model_info(
-    [
-        "spicyboros-7b-2.2",
-        "spicyboros-13b-2.2",
-        "spicyboros-70b-2.2",
-    ],
-    "spicyboros",
-    "https://huggingface.co/jondurbin/spicyboros-70b-2.2",
-    "de-aligned versions of the airoboros models",
-)
-
-register_model_info(
-    ["Robin-7b-v2", "Robin-13b-v2", "Robin-33b-v2"],
-    "Robin-v2",
-    "https://huggingface.co/OptimalScale/robin-7b-v2-delta",
-    "A chatbot fine-tuned from LLaMA-7b, achieving competitive performance on chitchat, commonsense reasoning and instruction-following tasks, by OptimalScale, HKUST.",
-)
-
-register_model_info(
-    ["manticore-13b-chat"],
-    "Manticore 13B Chat",
-    "https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg",
-    "A chatbot fine-tuned from LlaMa across several CoT and chat datasets.",
-)
-
-register_model_info(
-    ["redpajama-incite-7b-chat"],
-    "RedPajama-INCITE-7B-Chat",
-    "https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat",
-    "A chatbot fine-tuned from RedPajama-INCITE-7B-Base by Together",
-)
-
-register_model_info(
-    [
-        "falcon-7b",
-        "falcon-7b-instruct",
-        "falcon-40b",
-        "falcon-40b-instruct",
-        "falcon-180b",
-        "falcon-180b-chat",
-    ],
-    "Falcon",
-    "https://huggingface.co/tiiuae/falcon-180B",
-    "TII's flagship series of large language models",
-)
-
-register_model_info(
-    ["tigerbot-7b-sft"],
-    "Tigerbot",
-    "https://huggingface.co/TigerResearch/tigerbot-7b-sft",
-    "TigerBot is a large-scale language model (LLM) with multiple languages and tasks.",
-)
-
-register_model_info(
-    ["internlm-chat-7b", "internlm-chat-7b-8k"],
-    "InternLM",
-    "https://huggingface.co/internlm/internlm-chat-7b",
-    "InternLM is a multi-language large-scale language model (LLM), developed by SHLAB.",
-)
-
-register_model_info(
-    ["Qwen-7B-Chat"],
-    "Qwen",
-    "https://huggingface.co/Qwen/Qwen-7B-Chat",
-    "Qwen is a multi-language large-scale language model (LLM), developed by Damo Academy.",
-)
-
-register_model_info(
-    ["Llama2-Chinese-13b-Chat", "LLama2-Chinese-13B"],
-    "Llama2-Chinese",
-    "https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat",
-    "Llama2-Chinese is a multi-language large-scale language model (LLM), developed by FlagAlpha.",
-)
-
-register_model_info(
-    ["Chinese-Alpaca-2-7B", "Chinese-Alpaca-2-13B"],
-    "Chinese-Alpaca",
-    "https://huggingface.co/hfl/chinese-alpaca-2-13b",
-    "New extended Chinese vocabulary beyond Llama-2, open-sourcing the Chinese LLaMA-2 and Alpaca-2 LLMs.",
-)
-
-register_model_info(
-    ["Vigogne-2-7B-Instruct", "Vigogne-2-13B-Instruct"],
-    "Vigogne-Instruct",
-    "https://huggingface.co/bofenghuang/vigogne-2-7b-instruct",
-    "Vigogne-Instruct is a French large language model (LLM) optimized for instruction-following, developed by Bofeng Huang",
-)
-
-register_model_info(
-    ["Vigogne-2-7B-Chat", "Vigogne-2-13B-Chat"],
-    "Vigogne-Chat",
-    "https://huggingface.co/bofenghuang/vigogne-2-7b-chat",
-    "Vigogne-Chat is a French large language model (LLM) optimized for instruction-following and multi-turn dialogues, developed by Bofeng Huang",
-)
-
-register_model_info(
-    ["stable-vicuna-13B-HF"],
-    "stable-vicuna",
-    "https://huggingface.co/TheBloke/stable-vicuna-13B-HF",
-    "StableVicuna is a Vicuna model fine-tuned using RLHF via PPO on various conversational and instructional datasets.",
-)
-
-register_model_info(
-    ["deluxe-chat-v1", "deluxe-chat-v1.1", "deluxe-chat-v1.2"],
-    "DeluxeChat",
-    "",
-    "Deluxe Chat",
-)
-
-register_model_info(
-    [
-        "Xwin-LM-7B-V0.1",
-        "Xwin-LM-13B-V0.1",
-        "Xwin-LM-70B-V0.1",
-        "Xwin-LM-7B-V0.2",
-        "Xwin-LM-13B-V0.2",
-    ],
-    "Xwin-LM",
-    "https://github.com/Xwin-LM/Xwin-LM",
-    "Chat models developed by Xwin-LM team",
-)
-
-register_model_info(
-    ["lemur-70b-chat"],
-    "Lemur-Chat",
-    "https://huggingface.co/OpenLemur/lemur-70b-chat-v1",
-    "an openly accessible language model optimized for both natural language and coding capabilities ",
-)
-
-register_model_info(
-    ["Mistral-7B-OpenOrca"],
-    "Open-Orca",
-    "https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca",
-    "A fine-tune of [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) using [OpenOrca dataset](https://huggingface.co/datasets/Open-Orca/OpenOrca)",
-)
-
-register_model_info(
-    ["dolphin-2.2.1-mistral-7b"],
-    "dolphin-mistral",
-    "https://huggingface.co/ehartford/dolphin-2.2.1-mistral-7b",
-    "An uncensored fine-tuned Mistral 7B",
-)
-
-register_model_info(
-    [
-        "AquilaChat-7B",
-        "AquilaChat2-7B",
-        "AquilaChat2-34B",
-    ],
-    "Aquila-Chat",
-    "https://huggingface.co/BAAI/AquilaChat2-34B",
-    "Chat models developed by BAAI team",
-)
-
-register_model_info(
-    ["xDAN-L1-Chat-RL-v1"],
-    "xDAN-L1-Chat",
-    "https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1",
-    "A large language chat model created by xDAN-AI.",
-)
-
-register_model_info(
-    ["MetaMath-70B-V1.0", "MetaMath-7B-V1.0"],
-    "MetaMath",
-    "https://huggingface.co/meta-math",
-    "MetaMath is a finetune of Llama2 on [MetaMathQA](https://huggingface.co/datasets/meta-math/MetaMathQA) that specializes in mathematical reasoning.",
-)
-
-register_model_info(
-    ["Yuan2-2B-hf", "Yuan2-51B-hf", "Yuan2-102B-hf"],
-    "IEIYuan",
-    "https://huggingface.co/IEITYuan",
-    "Yuan2 is a Basemodel developed by IEI.",
-)

arena_elo/elo_rating/utils.py

@@ -3,12 +3,20 @@ import pytz
 import PIL
 import os 
 
+import sys
+sys.path.append('../')
+from model.model_registry import get_model_info
+
 def detect_language(text: str) -> str:
     """Detect the langauge of a string."""
-    import polyglot  # pip3 install polyglot pyicu pycld2
-    from polyglot.detect import Detector
-    from polyglot.detect.base import logger as polyglot_logger
-    import pycld2
+    try:
+        import polyglot  # pip3 install polyglot pyicu pycld2
+        from polyglot.detect import Detector
+        from polyglot.detect.base import logger as polyglot_logger
+        import pycld2
+    except ImportError as e:
+        print("Please install the required libraries: polyglot, pycld2: pip3 install polyglot pyicu pycld2")
+        exit(1)
 
     polyglot_logger.setLevel("ERROR")
 

arena_elo/results/20240809/elo_results_image_editing.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b54d460d93f22ee6e2519ccc82e4ebd64ec81987505760c428b593ff91a13368
-size 62455
+oid sha256:fe056dd818906f7d714c47fdba95fdcd01db8b8aa8490cfa0399cf2baab086e0
+size 63251

arena_elo/results/20240809/image_editing_leaderboard.csv

@@ -1,10 +1,10 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-MagicBrush,MagicBrush,1112.4518849527376,1116.3228088792275,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-InfEdit,InfEdit,1077.2249767545911,1077.0861206067664,CC BY-NC-ND 4.0,"University of Michigan, University of California, Berkeley",https://huggingface.co/spaces/sled-umich/InfEdit
-CosXLEdit,CosXLEdit,1064.682441262532,1065.599445502775,cosxl-nc-community,Stability AI,https://huggingface.co/spaces/multimodalart/cosxl
+MagicBrush,MagicBrush,1112.4518849527376,1116.3228088792275,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush/
+InfEdit,InfEdit,1077.2249767545911,1077.0861206067664,CC BY-NC-ND 4.0,"University of Michigan, University of California, Berkeley",https://sled-group.github.io/InfEdit/
+CosXLEdit,CosXLEdit,1064.682441262532,1065.599445502775,cosxl-nc-community,Stability AI,https://huggingface.co/stabilityai/cosxl
 InstructPix2Pix,InstructPix2Pix,1035.4054226880567,1033.2086681336118,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
 PNP,PNP,1001.4916130267786,1006.0747389504991,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-Prompt2prompt,Prompt2prompt,990.9755779591587,992.0796321838297,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-CycleDiffusion,CycleDiffusion,938.5637450408492,931.6722574678079,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
+Prompt2prompt,Prompt2prompt,990.9755779591587,992.0796321838297,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io/
+CycleDiffusion,CycleDiffusion,938.5637450408492,931.6722574678079,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion?tab=readme-ov-file
 SDEdit,SDEdit,925.0466906054369,923.4126211812364,MIT License,Stanford University,https://sde-image-editing.github.io
-Pix2PixZero,Pix2PixZero,854.1576477098594,854.5437070942457,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
+Pix2PixZero,Pix2PixZero,854.1576477098594,854.5437070942457,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io/

arena_elo/results/20240819/elo_results_video_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b831fc6bba773f7ee2c5dd7d226c681d322e38e83b3b282fc6130f80a0aada55
-size 65301
+oid sha256:7c42587eea0a948a9cbd5cca3b94e8141e70bcb9a0ae86646bc22973ea44332a
+size 66377

arena_elo/results/20240819/video_generation_leaderboard.csv

@@ -1,11 +1,11 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-StableVideoDiffusion,StableVideoDiffusion,1143.2191298083312,1147.5193110801638,stable-video-diffusion-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
-T2VTurbo,T2VTurbo,1092.4551860072625,1093.43771637836,cc-by-nc-4.0,"University of California, Santa Barbara",https://huggingface.co/jiachenli-ucsb/T2V-Turbo-VC2
+StableVideoDiffusion,StableVideoDiffusion,1143.2191298083312,1147.5193110801638,SVD-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
+T2V-Turbo,T2V-Turbo,1092.4551860072625,1093.43771637836,cc-by-nc-4.0,"University of California, Santa Barbara",https://github.com/Ji4chenLi/t2v-turbo
 VideoCrafter2,VideoCrafter2,1076.7797371219958,1079.3282160297892,Apache 2.0,Tencent AI Lab,https://ailab-cvc.github.io/videocrafter2/
 AnimateDiff,AnimateDiff,1071.9888858847867,1073.0390578256142,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v
-CogVideoX,CogVideoX,1046.417835182279,1048.070311411077,N/A,N/A,N/A
+CogVideoX,CogVideoX,1046.417835182279,1048.070311411077,CogVideoX LICENSE,THUDM,https://github.com/THUDM/CogVideo
 LaVie,LaVie,999.6270938177271,1001.6816790561326,Apache 2.0,Shanghai AI Lab,https://github.com/Vchitect/LaVie
-OpenSora12,OpenSora12,924.0203683794037,903.7447677401199,N/A,N/A,N/A
+OpenSora v1.2,OpenSora v1.2,924.0203683794037,903.7447677401199,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
 OpenSora,OpenSora,916.0498767986071,917.9961732110512,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
 ModelScope,ModelScope,865.3398542588171,868.9309810684535,cc-by-nc-4.0,Alibaba Group,https://arxiv.org/abs/2308.06571
-AnimateDiffTurbo,AnimateDiffTurbo,864.1020327407906,866.2517861992403,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo
+AnimateDiff Turbo,AnimateDiff Turbo,864.1020327407906,866.2517861992403,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo

arena_elo/results/20240820/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9616bb1c5e8f8e239439869a0f396b5b98ed9f96a82d8da1f84ef9858fd18408
-size 86085
+oid sha256:a55e58f091b88f7cc16382c56b7d655cbcd86bd23c3f3de214ef5484edfd1e4d
+size 88247

arena_elo/results/20240820/t2i_generation_leaderboard.csv

@@ -1,18 +1,18 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGround V2.5,PlayGround V2.5,1134.1072448268308,1134.2706104849465,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-FLUX1dev,FLUX1dev,1123.2199818444176,1141.2729049211114,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
-PlayGround V2,PlayGround V2,1077.6606831940005,1074.8256920086785,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-FLUX1schnell,FLUX1schnell,1065.8699453260276,1062.2080199364366,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
-HunyuanDiT,HunyuanDiT,1054.0890297393096,1039.1419749190748,tencent-hunyuan-community,Tencent,https://huggingface.co/Tencent-Hunyuan/HunyuanDiT
-StableCascade,StableCascade,1041.412322556985,1043.2756353067755,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-AuraFlow,AuraFlow,1028.5655063021388,1021.7707536233816,Apache-2.0,Fal.AI,https://huggingface.co/fal/AuraFlow
-PixArtAlpha,PixArtAlpha,1027.9698284008514,1016.530964819809,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXLLightning,SDXLLightning,1027.9052590104445,1030.69166244485,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtSigma,PixArtSigma,1022.8987259151355,1021.0198915432052,openrail++,PixArt-alpha,https://fal.ai/models/fal-ai/pixart-sigma
-SD3,SD3,992.8340158341957,987.2147159388242,stabilityai-nc-research-community,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-3-medium
-Kolors,Kolors,980.6259278504377,977.8827108575291,Apache-2.0,Kwai Kolors,https://huggingface.co/Kwai-Kolors/Kolors
-SDXL,SDXL,968.880958203862,967.8253582602699,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,917.5724578816604,913.3288656646328,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-LCM(v1.5/XL),LCM(v1.5/XL),909.8725596407722,902.285091713656,openrail++,Latent Consistency,https://fal.ai/models/fal-ai/fast-lcm-diffusion/api
-OpenJourney,OpenJourney,832.2754474721498,825.5234798053013,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.2401060007853,806.4811434730161,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
+PlayGround V2.5,PlayGround V2.5,1132.890132362509,1133.2132994392719,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
+FLUX.1-dev,FLUX.1-dev,1130.7155148553181,1147.438205902597,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+PlayGround V2,PlayGround V2,1076.980283311089,1074.3249013429554,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+FLUX.1-schnell,FLUX.1-schnell,1066.4020402818517,1062.8730931830312,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+HunyuanDiT,HunyuanDiT,1053.572229850288,1038.703774370632,tencent-hunyuan-community,Tencent,https://github.com/Tencent/HunyuanDiT
+StableCascade,StableCascade,1040.7649360610326,1042.7543257183775,stable-cascade-nc-community (other),Stability AI,https://fal.ai/models/stable-cascade/api
+AuraFlow,AuraFlow,1028.6476636951288,1021.8888350731722,Apache-2.0,Fal.AI,https://huggingface.co/fal/AuraFlow
+PixArtAlpha,PixArtAlpha,1027.5984425989943,1016.2046020683781,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL-Lightning,SDXL-Lightning,1027.28056074664,1030.1892737463068,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtSigma,PixArtSigma,1022.278191734517,1020.5244848333533,openrail++,PixArt-alpha,https://github.com/PixArt-alpha/PixArt-sigma
+SD3,SD3,992.4683734086498,986.8567562452171,stabilityai-nc-research-community,Stability AI,https://huggingface.co/blog/sd3
+Kolors,Kolors,980.5720834009985,977.940378018964,Apache-2.0,Kwai Kolors,https://huggingface.co/Kwai-Kolors/Kolors
+SDXL,SDXL,968.286211097592,967.3440457396239,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,916.9664722660239,912.8537618702409,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+LCM(v1.5/XL),LCM(v1.5/XL),909.2587727377047,901.7564308088253,openrail++,Latent Consistency,https://fal.ai/models/fast-lcm-diffusion-turbo
+OpenJourney,OpenJourney,831.6663894963693,825.0727531727863,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,793.6517020952907,806.0428762517084,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/latest/elo_results_image_editing.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b54d460d93f22ee6e2519ccc82e4ebd64ec81987505760c428b593ff91a13368
-size 62455
+oid sha256:fe056dd818906f7d714c47fdba95fdcd01db8b8aa8490cfa0399cf2baab086e0
+size 63251

arena_elo/results/latest/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9616bb1c5e8f8e239439869a0f396b5b98ed9f96a82d8da1f84ef9858fd18408
-size 86085
+oid sha256:a55e58f091b88f7cc16382c56b7d655cbcd86bd23c3f3de214ef5484edfd1e4d
+size 88247

arena_elo/results/latest/elo_results_video_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b831fc6bba773f7ee2c5dd7d226c681d322e38e83b3b282fc6130f80a0aada55
-size 65301
+oid sha256:7c42587eea0a948a9cbd5cca3b94e8141e70bcb9a0ae86646bc22973ea44332a
+size 66377

arena_elo/results/latest/image_editing_leaderboard.csv

@@ -1,10 +1,10 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-MagicBrush,MagicBrush,1112.4518849527376,1116.3228088792275,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush
-InfEdit,InfEdit,1077.2249767545911,1077.0861206067664,CC BY-NC-ND 4.0,"University of Michigan, University of California, Berkeley",https://huggingface.co/spaces/sled-umich/InfEdit
-CosXLEdit,CosXLEdit,1064.682441262532,1065.599445502775,cosxl-nc-community,Stability AI,https://huggingface.co/spaces/multimodalart/cosxl
+MagicBrush,MagicBrush,1112.4518849527376,1116.3228088792275,CC-BY-4.0,"The Ohio State University, University of Waterloo",https://osu-nlp-group.github.io/MagicBrush/
+InfEdit,InfEdit,1077.2249767545911,1077.0861206067664,CC BY-NC-ND 4.0,"University of Michigan, University of California, Berkeley",https://sled-group.github.io/InfEdit/
+CosXLEdit,CosXLEdit,1064.682441262532,1065.599445502775,cosxl-nc-community,Stability AI,https://huggingface.co/stabilityai/cosxl
 InstructPix2Pix,InstructPix2Pix,1035.4054226880567,1033.2086681336118,"Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros","University of California, Berkeley",https://www.timothybrooks.com/instruct-pix2pix
 PNP,PNP,1001.4916130267786,1006.0747389504991,-,Weizmann Institute of Science,https://github.com/MichalGeyer/plug-and-play
-Prompt2prompt,Prompt2prompt,990.9755779591587,992.0796321838297,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io
-CycleDiffusion,CycleDiffusion,938.5637450408492,931.6722574678079,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion
+Prompt2prompt,Prompt2prompt,990.9755779591587,992.0796321838297,Apache-2.0,"Google, Tel Aviv University",https://prompt-to-prompt.github.io/
+CycleDiffusion,CycleDiffusion,938.5637450408492,931.6722574678079,X11,Carnegie Mellon University,https://github.com/ChenWu98/cycle-diffusion?tab=readme-ov-file
 SDEdit,SDEdit,925.0466906054369,923.4126211812364,MIT License,Stanford University,https://sde-image-editing.github.io
-Pix2PixZero,Pix2PixZero,854.1576477098594,854.5437070942457,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io
+Pix2PixZero,Pix2PixZero,854.1576477098594,854.5437070942457,MIT License,"Carnegie Mellon University, Adobe Research",https://pix2pixzero.github.io/

arena_elo/results/latest/t2i_generation_leaderboard.csv

@@ -1,18 +1,18 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGround V2.5,PlayGround V2.5,1134.1072448268308,1134.2706104849465,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
-FLUX1dev,FLUX1dev,1123.2199818444176,1141.2729049211114,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
-PlayGround V2,PlayGround V2,1077.6606831940005,1074.8256920086785,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-FLUX1schnell,FLUX1schnell,1065.8699453260276,1062.2080199364366,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
-HunyuanDiT,HunyuanDiT,1054.0890297393096,1039.1419749190748,tencent-hunyuan-community,Tencent,https://huggingface.co/Tencent-Hunyuan/HunyuanDiT
-StableCascade,StableCascade,1041.412322556985,1043.2756353067755,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-AuraFlow,AuraFlow,1028.5655063021388,1021.7707536233816,Apache-2.0,Fal.AI,https://huggingface.co/fal/AuraFlow
-PixArtAlpha,PixArtAlpha,1027.9698284008514,1016.530964819809,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXLLightning,SDXLLightning,1027.9052590104445,1030.69166244485,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtSigma,PixArtSigma,1022.8987259151355,1021.0198915432052,openrail++,PixArt-alpha,https://fal.ai/models/fal-ai/pixart-sigma
-SD3,SD3,992.8340158341957,987.2147159388242,stabilityai-nc-research-community,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-3-medium
-Kolors,Kolors,980.6259278504377,977.8827108575291,Apache-2.0,Kwai Kolors,https://huggingface.co/Kwai-Kolors/Kolors
-SDXL,SDXL,968.880958203862,967.8253582602699,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,917.5724578816604,913.3288656646328,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-LCM(v1.5/XL),LCM(v1.5/XL),909.8725596407722,902.285091713656,openrail++,Latent Consistency,https://fal.ai/models/fal-ai/fast-lcm-diffusion/api
-OpenJourney,OpenJourney,832.2754474721498,825.5234798053013,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.2401060007853,806.4811434730161,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
+PlayGround V2.5,PlayGround V2.5,1132.890132362509,1133.2132994392719,Playground v2.5 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
+FLUX.1-dev,FLUX.1-dev,1130.7155148553181,1147.438205902597,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+PlayGround V2,PlayGround V2,1076.980283311089,1074.3249013429554,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
+FLUX.1-schnell,FLUX.1-schnell,1066.4020402818517,1062.8730931830312,flux-1-dev-non-commercial-license (other),Black Forest Labs,https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
+HunyuanDiT,HunyuanDiT,1053.572229850288,1038.703774370632,tencent-hunyuan-community,Tencent,https://github.com/Tencent/HunyuanDiT
+StableCascade,StableCascade,1040.7649360610326,1042.7543257183775,stable-cascade-nc-community (other),Stability AI,https://fal.ai/models/stable-cascade/api
+AuraFlow,AuraFlow,1028.6476636951288,1021.8888350731722,Apache-2.0,Fal.AI,https://huggingface.co/fal/AuraFlow
+PixArtAlpha,PixArtAlpha,1027.5984425989943,1016.2046020683781,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL-Lightning,SDXL-Lightning,1027.28056074664,1030.1892737463068,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtSigma,PixArtSigma,1022.278191734517,1020.5244848333533,openrail++,PixArt-alpha,https://github.com/PixArt-alpha/PixArt-sigma
+SD3,SD3,992.4683734086498,986.8567562452171,stabilityai-nc-research-community,Stability AI,https://huggingface.co/blog/sd3
+Kolors,Kolors,980.5720834009985,977.940378018964,Apache-2.0,Kwai Kolors,https://huggingface.co/Kwai-Kolors/Kolors
+SDXL,SDXL,968.286211097592,967.3440457396239,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,916.9664722660239,912.8537618702409,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+LCM(v1.5/XL),LCM(v1.5/XL),909.2587727377047,901.7564308088253,openrail++,Latent Consistency,https://fal.ai/models/fast-lcm-diffusion-turbo
+OpenJourney,OpenJourney,831.6663894963693,825.0727531727863,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,793.6517020952907,806.0428762517084,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/latest/video_generation_leaderboard.csv

@@ -1,11 +1,11 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
 StableVideoDiffusion,StableVideoDiffusion,1143.2191298083312,1147.5193110801638,SVD-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
-T2VTurbo,T2VTurbo,1092.4551860072625,1093.43771637836,cc-by-nc-4.0,"University of California, Santa Barbara",https://huggingface.co/jiachenli-ucsb/T2V-Turbo-VC2
+T2V-Turbo,T2V-Turbo,1092.4551860072625,1093.43771637836,cc-by-nc-4.0,"University of California, Santa Barbara",https://github.com/Ji4chenLi/t2v-turbo
 VideoCrafter2,VideoCrafter2,1076.7797371219958,1079.3282160297892,Apache 2.0,Tencent AI Lab,https://ailab-cvc.github.io/videocrafter2/
 AnimateDiff,AnimateDiff,1071.9888858847867,1073.0390578256142,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v
-CogVideoX,CogVideoX,1046.417835182279,1048.070311411077,CogVideoX License,Tsinghua University,https://huggingface.co/THUDM/CogVideoX-2b
+CogVideoX,CogVideoX,1046.417835182279,1048.070311411077,CogVideoX LICENSE,THUDM,https://github.com/THUDM/CogVideo
 LaVie,LaVie,999.6270938177271,1001.6816790561326,Apache 2.0,Shanghai AI Lab,https://github.com/Vchitect/LaVie
-OpenSora12,OpenSora12,924.0203683794037,903.7447677401199,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
+OpenSora v1.2,OpenSora v1.2,924.0203683794037,903.7447677401199,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
 OpenSora,OpenSora,916.0498767986071,917.9961732110512,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
 ModelScope,ModelScope,865.3398542588171,868.9309810684535,cc-by-nc-4.0,Alibaba Group,https://arxiv.org/abs/2308.06571
-AnimateDiffTurbo,AnimateDiffTurbo,864.1020327407906,866.2517861992403,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo
+AnimateDiff Turbo,AnimateDiff Turbo,864.1020327407906,866.2517861992403,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo

arena_elo/update_elo_rating.sh

@@ -48,17 +48,14 @@ mv ./elo_results_$video_generation_battle_cutoff_date.pkl ./results/$video_gener
 # generat the leaderboard
 
 python -m elo_rating.generate_leaderboard \
-    --model_info_file "./edition_model_info.json" \
     --elo_rating_pkl "./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl" \
     --output_csv "./results/$edition_battle_cutoff_date/image_editing_leaderboard.csv"
 
 python -m elo_rating.generate_leaderboard \
-    --model_info_file "./generation_model_info.json" \
     --elo_rating_pkl "./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl" \
     --output_csv "./results/$generation_battle_cutoff_date/t2i_generation_leaderboard.csv"
 
 python -m elo_rating.generate_leaderboard \
-    --model_info_file "./video_generation_model_info.json" \
     --elo_rating_pkl "./results/$video_generation_battle_cutoff_date/elo_results_video_generation.pkl" \
     --output_csv "./results/$video_generation_battle_cutoff_date/video_generation_leaderboard.csv"
 

model/model_registry.py

@@ -1,16 +1,42 @@
 from collections import namedtuple
 from typing import List
 
-ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description"])
+IMAGE_GENERATION_MODELS = ['imagenhub_SDXLTurbo_generation','imagenhub_SDXL_generation', 'imagenhub_PixArtAlpha_generation', 'imagenhub_PixArtSigma_generation',
+                           'imagenhub_OpenJourney_generation','imagenhub_SDXLLightning_generation', 'imagenhub_StableCascade_generation', 'imagenhub_HunyuanDiT_generation',
+                           'playground_PlayGroundV2.5_generation', 'imagenhub_Kolors_generation', 'imagenhub_SD3_generation',
+                           'fal_AuraFlow_text2image', 'fal_FLUX1schnell_text2image', 'fal_FLUX1dev_text2image']  # 'playground_PlayGroundV2_generation'
+IMAGE_EDITION_MODELS = ['imagenhub_CycleDiffusion_edition', 'imagenhub_Pix2PixZero_edition', 'imagenhub_Prompt2prompt_edition',
+                        'imagenhub_SDEdit_edition', 'imagenhub_InstructPix2Pix_edition', 
+                        'imagenhub_MagicBrush_edition', 'imagenhub_PNP_edition', 
+                        'imagenhub_InfEdit_edition', 'imagenhub_CosXLEdit_edition', 'imagenhub_UltraEdit_edition']
+VIDEO_GENERATION_MODELS = ['fal_AnimateDiff_text2video',
+                           'fal_AnimateDiffTurbo_text2video',
+                           #'videogenhub_LaVie_generation', 
+                           'videogenhub_VideoCrafter2_generation',
+                           #'videogenhub_ModelScope_generation', 
+                           'videogenhub_CogVideoX_generation', 'videogenhub_OpenSora12_generation',
+                           #'videogenhub_OpenSora_generation', 
+                           #'videogenhub_T2VTurbo_generation',
+                           'fal_T2VTurbo_text2video',
+                           'fal_StableVideoDiffusion_text2video']
+MUSEUM_UNSUPPORTED_MODELS = ['videogenhub_OpenSoraPlan_generation']
+DESIRED_APPEAR_MODEL = ['videogenhub_T2VTurbo_generation','fal_StableVideoDiffusion_text2video']
+
+ALL_MODELS = IMAGE_GENERATION_MODELS + IMAGE_EDITION_MODELS + VIDEO_GENERATION_MODELS
+
+
+ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description", "license", "organization", "type"])
 model_info = {}
 
 def register_model_info(
-    full_names: List[str], simple_name: str, link: str, description: str
+    full_names: List[str], simple_name: str, link: str, description: str,
+    license: str, organization: str, model_type: str
 ):
-    info = ModelInfo(simple_name, link, description)
-
+    info = ModelInfo(simple_name, link, description, license, organization, model_type)
     for full_name in full_names:
         model_info[full_name] = info
+        model_info[full_name.split("_")[1]] = info
+    model_info[simple_name] = info
 
 def get_model_info(name: str) -> ModelInfo:
     if name in model_info:
@@ -18,7 +44,8 @@ def get_model_info(name: str) -> ModelInfo:
     else:
         # To fix this, please use `register_model_info` to register your model
         return ModelInfo(
-            name, "", "Register the description at fastchat/model/model_registry.py"
+            name, "-", "Register the description at fastchat/model/model_registry.py",
+            "-", "-", None
         )
 
 def get_model_description_md(model_list):
@@ -50,6 +77,9 @@ register_model_info(
     "LCM",
     "https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7",
     "Latent Consistency Models.",
+    "MIT License",
+    "Tsinghua University",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -57,27 +87,39 @@ register_model_info(
     "LCM(v1.5/XL)",
     "https://fal.ai/models/fast-lcm-diffusion-turbo",
     "Latent Consistency Models (v1.5/XL)",
+    "openrail++",
+    "Latent Consistency",
+    "text2image_generation"
 )
 
 register_model_info(
     ["imagenhub_PlayGroundV2_generation", 'playground_PlayGroundV2_generation'],
-    "Playground v2",
+    "PlayGround V2",
     "https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic",
     "Playground v2 – 1024px Aesthetic Model",
+    "Playground v2 Community License",
+    "Playground",
+    "text2image_generation"
 )
 
 register_model_info(
     ["imagenhub_PlayGroundV2.5_generation", 'playground_PlayGroundV2.5_generation'],
-    "Playground v2.5",
+    "PlayGround V2.5",
     "https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic",
     "Playground v2.5 is the state-of-the-art open-source model in aesthetic quality",
+    "Playground v2.5 Community License",
+    "Playground",
+    "text2image_generation"
 )
 
 register_model_info(
     ["imagenhub_OpenJourney_generation"],
-    "Openjourney",
+    "OpenJourney",
     "https://huggingface.co/prompthero/openjourney",
     "Openjourney is an open source Stable Diffusion fine tuned model on Midjourney images, by PromptHero.",
+    "creativeml-openrail-m",
+    "PromptHero",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -85,6 +127,19 @@ register_model_info(
     "SDXLTurbo",
     "https://huggingface.co/stabilityai/sdxl-turbo",
     "SDXL-Turbo is a fast generative text-to-image model.",
+    "sai-nc-community (other)",
+    "Stability AI",
+    "text2image_generation"
+)
+
+register_model_info(
+    ["imagenhub_SDEdit_edition"],
+    "SDEdit",
+    "https://sde-image-editing.github.io",
+    "SDEdit is an image synthesis and editing framework based on stochastic differential equations (SDEs) or diffusion models.",
+    "MIT License",
+    "Stanford University",
+    "image_edition"
 )
 
 register_model_info(
@@ -92,6 +147,9 @@ register_model_info(
     "SDXL",
     "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
     "SDXL is a Latent Diffusion Model that uses two fixed, pretrained text encoders.",
+    "openrail++",
+    "Stability AI",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -99,6 +157,9 @@ register_model_info(
     "SD3",
     "https://huggingface.co/blog/sd3",
     "SD3 is a novel Multimodal Diffusion Transformer (MMDiT) model.",
+    "stabilityai-nc-research-community",
+    "Stability AI",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -106,6 +167,9 @@ register_model_info(
     "PixArtAlpha",
     "https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS",
     "Pixart-α consists of pure transformer blocks for latent diffusion.",
+    "openrail++",
+    "PixArt-alpha",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -113,6 +177,9 @@ register_model_info(
     "PixArtSigma",
     "https://github.com/PixArt-alpha/PixArt-sigma",
     "Improved version of Pixart-α.",
+    "openrail++",
+    "PixArt-alpha",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -120,6 +187,9 @@ register_model_info(
     "SDXL-Lightning",
     "https://huggingface.co/ByteDance/SDXL-Lightning",
     "SDXL-Lightning is a lightning-fast text-to-image generation model.",
+    "openrail++",
+    "ByteDance",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -127,6 +197,9 @@ register_model_info(
     "StableCascade",
     "https://huggingface.co/stabilityai/stable-cascade",
     "StableCascade is built upon the Würstchen architecture and working at a much smaller latent space.",
+    "stable-cascade-nc-community (other)",
+    "Stability AI",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -134,6 +207,9 @@ register_model_info(
     "HunyuanDiT",
     "https://github.com/Tencent/HunyuanDiT",
     "HunyuanDiT is a Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding",
+    "tencent-hunyuan-community",
+    "Tencent",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -141,6 +217,9 @@ register_model_info(
     "Kolors",
     "https://huggingface.co/Kwai-Kolors/Kolors",
     "Kolors is a large-scale text-to-image generation model based on latent diffusion",
+    "Apache-2.0",
+    "Kwai Kolors",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -148,6 +227,9 @@ register_model_info(
     "AuraFlow",
     "https://huggingface.co/fal/AuraFlow",
     "Opensourced flow-based text-to-image generation model.",
+    "Apache-2.0",
+    "Fal.AI",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -155,6 +237,9 @@ register_model_info(
     "FLUX.1-schnell",
     "https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux",
     "Flux is a series of text-to-image generation models based on diffusion transformers. Timestep-distilled version.",
+    "flux-1-dev-non-commercial-license (other)",
+    "Black Forest Labs",
+    "text2image_generation"
 )
 
 register_model_info(
@@ -162,6 +247,9 @@ register_model_info(
     "FLUX.1-dev",
     "https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux",
     "Flux is a series of text-to-image generation models based on diffusion transformers. Guidance-distilled version.",
+    "flux-1-dev-non-commercial-license (other)",
+    "Black Forest Labs",
+    "text2image_generation"
 )
 
 
@@ -171,6 +259,9 @@ register_model_info(
     "CycleDiffusion",
     "https://github.com/ChenWu98/cycle-diffusion?tab=readme-ov-file",
     "A latent space for stochastic diffusion models.",
+    "X11",
+    "Carnegie Mellon University",
+    "image_edition"
 )
 
 register_model_info(
@@ -178,6 +269,9 @@ register_model_info(
     "Pix2PixZero",
     "https://pix2pixzero.github.io/",
     "A zero-shot Image-to-Image translation model.",
+    "MIT License",
+    "Carnegie Mellon University, Adobe Research",
+    "image_edition"
 )
 
 register_model_info(
@@ -185,6 +279,9 @@ register_model_info(
     "Prompt2prompt",
     "https://prompt-to-prompt.github.io/",
     "Image Editing with Cross-Attention Control.",
+    "Apache-2.0",
+    "Google, Tel Aviv University",
+    "image_edition"
 )
 
 
@@ -193,6 +290,9 @@ register_model_info(
     "InstructPix2Pix",
     "https://www.timothybrooks.com/instruct-pix2pix",
     "An instruction-based image editing model.",
+    "Copyright 2023 Timothy Brooks, Aleksander Holynski, Alexei A. Efros",
+    "University of California, Berkeley",
+    "image_edition"
 )
 
 register_model_info(
@@ -200,6 +300,9 @@ register_model_info(
     "MagicBrush",
     "https://osu-nlp-group.github.io/MagicBrush/",
     "Manually Annotated Dataset for Instruction-Guided Image Editing.",
+    "CC-BY-4.0",
+    "The Ohio State University, University of Waterloo",
+    "image_edition"
 )
 
 register_model_info(
@@ -207,6 +310,9 @@ register_model_info(
     "PNP",
     "https://github.com/MichalGeyer/plug-and-play",
     "Plug-and-Play Diffusion Features for Text-Driven Image-to-Image Translation.",
+    "-",
+    "Weizmann Institute of Science",
+    "image_edition"
 )
 
 register_model_info(
@@ -214,6 +320,9 @@ register_model_info(
     "InfEdit",
     "https://sled-group.github.io/InfEdit/",
     "Inversion-Free Image Editing with Natural Language.",
+    "CC BY-NC-ND 4.0",
+    "University of Michigan, University of California, Berkeley",
+    "image_edition"
 )
 
 register_model_info(
@@ -221,6 +330,9 @@ register_model_info(
     "CosXLEdit",
     "https://huggingface.co/stabilityai/cosxl",
     "An instruction-based image editing model from SDXL.",
+    "cosxl-nc-community",
+    "Stability AI",
+    "image_edition"
 )
 
 register_model_info(
@@ -228,6 +340,9 @@ register_model_info(
     "UltraEdit",
     "https://ultra-editing.github.io/",
     "Instruction-based Fine-Grained Image Editing at Scale.",
+    "other",
+    "Peking University; BIGAI",
+    "image_edition"
 )
 
 register_model_info(
@@ -235,6 +350,9 @@ register_model_info(
     "StableCascade",
     "https://fal.ai/models/stable-cascade/api",
     "StableCascade is a generative model that can generate high-quality images from text prompts.",
+    "stable-cascade-nc-community (other)",
+    "Stability AI",
+    "image_edition"
 )
 
 register_model_info(
@@ -242,6 +360,9 @@ register_model_info(
     "AnimateDiff",
     "https://fal.ai/models/fast-animatediff-t2v",
     "AnimateDiff is a text-driven models that produce diverse and personalized animated images.",
+    "creativeml-openrail-m",
+    "The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",
+    "text2video_generation"
 )
 
 register_model_info(
@@ -249,6 +370,9 @@ register_model_info(
     "StableVideoDiffusion",
     "https://fal.ai/models/fal-ai/fast-svd/text-to-video/api",
     "Stable Video Diffusion empowers individuals to transform text and image inputs into vivid scenes.",
+    "SVD-nc-community",
+    "Stability AI",
+    "text2video_generation"
 )
 
 register_model_info(
@@ -256,6 +380,9 @@ register_model_info(
     "AnimateDiff Turbo",
     "https://fal.ai/models/fast-animatediff-t2v-turbo",
     "AnimateDiff Turbo is a lightning version of AnimateDiff.",
+    "creativeml-openrail-m",
+    "The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",
+    "text2video_generation"
 )
 
 register_model_info(
@@ -263,20 +390,28 @@ register_model_info(
     "VideoCrafter2",
     "https://ailab-cvc.github.io/videocrafter2/",
     "VideoCrafter2 is a T2V model that disentangling motion from appearance.",
+    "Apache 2.0",
+    "Tencent AI Lab",
+    "text2video_generation"
 )
 
-"""
 register_model_info(
     ["videogenhub_LaVie_generation"],
     "LaVie",
     "https://github.com/Vchitect/LaVie",
     "LaVie is a video generation model with cascaded latent diffusion models.",
+    "Apache 2.0",
+    "Shanghai AI Lab",
+    "text2video_generation"
 )
 register_model_info(
     ["videogenhub_ModelScope_generation"],
     "ModelScope",
     "https://arxiv.org/abs/2308.06571",
     "ModelScope is a a T2V synthesis model that evolves from a T2I synthesis model.",
+    "cc-by-nc-4.0",
+    "Alibaba Group",
+    "text2video_generation"
 )
 
 register_model_info(
@@ -284,13 +419,19 @@ register_model_info(
     "OpenSora",
     "https://github.com/hpcaitech/Open-Sora",
     "A community-driven opensource implementation of Sora.",
+    "Apache 2.0",
+    "HPC-AI Tech",
+    "text2video_generation"
 )
-"""
+
 register_model_info(
     ["videogenhub_OpenSora12_generation"],
     "OpenSora v1.2",
     "https://github.com/hpcaitech/Open-Sora",
     "A community-driven opensource implementation of Sora. v1.2",
+    "Apache 2.0",
+    "HPC-AI Tech",
+    "text2video_generation"
 )
 
 register_model_info(
@@ -298,6 +439,9 @@ register_model_info(
     "CogVideoX",
     "https://github.com/THUDM/CogVideo",
     "Text-to-Video Diffusion Models with An Expert Transformer.",
+    "CogVideoX LICENSE",
+    "THUDM",
+    "text2video_generation"
 )
     
 register_model_info(
@@ -305,5 +449,10 @@ register_model_info(
     "T2V-Turbo",
     "https://github.com/Ji4chenLi/t2v-turbo",
     "Video Consistency Model with Mixed Reward Feedback.",
+    "cc-by-nc-4.0",
+    "University of California, Santa Barbara",
+    "text2video_generation"
 )
     
+
+assert all([model in model_info for model in ALL_MODELS]), "Some models are not registered in model_info"

model/models/__init__.py

@@ -2,6 +2,7 @@ from .imagenhub_models import load_imagenhub_model
 from .playground_api import load_playground_model
 from .fal_api_models import load_fal_model
 from .videogenhub_models import load_videogenhub_model
+from ..model_registry import model_info
 
 
 # IMAGE_GENERATION_MODELS = ['fal_LCM(v1.5/XL)_text2image','fal_SDXLTurbo_text2image','fal_SDXL_text2image', 'imagenhub_PixArtAlpha_generation', 'fal_PixArtSigma_text2image',
@@ -28,6 +29,12 @@ VIDEO_GENERATION_MODELS = ['fal_AnimateDiff_text2video',
 MUSEUM_UNSUPPORTED_MODELS = ['videogenhub_OpenSoraPlan_generation']
 DESIRED_APPEAR_MODEL = ['videogenhub_T2VTurbo_generation','fal_StableVideoDiffusion_text2video']
 
+ALL_MODELS = IMAGE_GENERATION_MODELS + IMAGE_EDITION_MODELS + VIDEO_GENERATION_MODELS
+
+missing_models = [model for model in ALL_MODELS if model not in model_info]
+if missing_models:
+    raise ValueError(f"Missing models in model_info: {missing_models}")
+
 def load_pipeline(model_name):
     """
     Load a model pipeline based on the model name