update serve

arena_elo/elo_rating/basic_stats.py

@@ -0,0 +1,227 @@
+import argparse
+import code
+import datetime
+import json
+import os
+from pytz import timezone
+import time
+
+import pandas as pd  # pandas>=2.0.3
+import plotly.express as px
+import plotly.graph_objects as go
+from tqdm import tqdm
+
+NUM_SERVERS = 1
+LOG_ROOT_DIR = os.getenv("LOGDIR", None)
+if LOG_ROOT_DIR is None:
+    raise ValueError("LOGDIR environment variable not set, please set it by `export LOGDIR=...`")
+
+def get_log_files(max_num_files=None):
+    log_root = os.path.expanduser(LOG_ROOT_DIR)
+    filenames = []
+    if NUM_SERVERS == 1:
+        for filename in os.listdir(log_root):
+            if filename.endswith("-conv.json"):
+                filepath = f"{log_root}/{filename}"
+                name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
+                filenames.append(name_tstamp_tuple)
+    else:
+        for i in range(NUM_SERVERS):
+            for filename in os.listdir(f"{log_root}/server{i}"):
+                if filename.endswith("-conv.json"):
+                    filepath = f"{log_root}/server{i}/{filename}"
+                    name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
+                    filenames.append(name_tstamp_tuple)
+    # sort by tstamp
+    filenames = sorted(filenames, key=lambda x: x[1])
+    filenames = [x[0] for x in filenames]
+
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def load_log_files(filename):
+    data = []
+    for retry in range(5):
+        try:
+            lines = open(filename).readlines()
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+
+    for l in lines:
+        row = json.loads(l)
+        data.append(
+            dict(
+                type=row["type"],
+                tstamp=row["tstamp"],
+                model=row.get("model", ""),
+                models=row.get("models", ["", ""]),
+            )
+        )
+    return data
+
+
+def load_log_files_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(load_log_files, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+
+def get_anony_vote_df(df):
+    anony_vote_df = df[
+        df["type"].isin(["leftvote", "rightvote", "tievote", "bothbad_vote"])
+    ]
+    anony_vote_df = anony_vote_df[anony_vote_df["models"].apply(lambda x: x[0] == "")]
+    return anony_vote_df
+
+
+def merge_counts(series, on, names):
+    ret = pd.merge(series[0], series[1], on=on)
+    for i in range(2, len(series)):
+        ret = pd.merge(ret, series[i], on=on)
+    ret = ret.reset_index()
+    old_names = list(ret.columns)[-len(series) :]
+    rename = {old_name: new_name for old_name, new_name in zip(old_names, names)}
+    ret = ret.rename(columns=rename)
+    return ret
+
+
+def report_basic_stats(log_files):
+    df_all = load_log_files_parallel(log_files)
+    df_all = pd.DataFrame(df_all)
+    now_t = df_all["tstamp"].max()
+    df_1_hour = df_all[df_all["tstamp"] > (now_t - 3600)]
+    df_1_day = df_all[df_all["tstamp"] > (now_t - 3600 * 24)]
+    anony_vote_df_all = get_anony_vote_df(df_all)
+
+    # Chat trends
+    chat_dates = [
+        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
+            "%Y-%m-%d"
+        )
+        for x in df_all[df_all["type"] == "chat"]["tstamp"]
+    ]
+    chat_dates_counts = pd.value_counts(chat_dates)
+    vote_dates = [
+        datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime(
+            "%Y-%m-%d"
+        )
+        for x in anony_vote_df_all["tstamp"]
+    ]
+    vote_dates_counts = pd.value_counts(vote_dates)
+    chat_dates_bar = go.Figure(
+        data=[
+            go.Bar(
+                name="Anony. Vote",
+                x=vote_dates_counts.index,
+                y=vote_dates_counts,
+                text=[f"{val:.0f}" for val in vote_dates_counts],
+                textposition="auto",
+            ),
+            go.Bar(
+                name="Chat",
+                x=chat_dates_counts.index,
+                y=chat_dates_counts,
+                text=[f"{val:.0f}" for val in chat_dates_counts],
+                textposition="auto",
+            ),
+        ]
+    )
+    chat_dates_bar.update_layout(
+        barmode="stack",
+        xaxis_title="Dates",
+        yaxis_title="Count",
+        height=300,
+        width=1200,
+    )
+
+    # Model call counts
+    model_hist_all = df_all[df_all["type"] == "chat"]["model"].value_counts()
+    model_hist_1_day = df_1_day[df_1_day["type"] == "chat"]["model"].value_counts()
+    model_hist_1_hour = df_1_hour[df_1_hour["type"] == "chat"]["model"].value_counts()
+    model_hist = merge_counts(
+        [model_hist_all, model_hist_1_day, model_hist_1_hour],
+        on="model",
+        names=["All", "Last Day", "Last Hour"],
+    )
+    model_hist_md = model_hist.to_markdown(index=False, tablefmt="github")
+
+    # Action counts
+    action_hist_all = df_all["type"].value_counts()
+    action_hist_1_day = df_1_day["type"].value_counts()
+    action_hist_1_hour = df_1_hour["type"].value_counts()
+    action_hist = merge_counts(
+        [action_hist_all, action_hist_1_day, action_hist_1_hour],
+        on="type",
+        names=["All", "Last Day", "Last Hour"],
+    )
+    action_hist_md = action_hist.to_markdown(index=False, tablefmt="github")
+
+    # Anony vote counts
+    anony_vote_hist_all = anony_vote_df_all["type"].value_counts()
+    anony_vote_df_1_day = get_anony_vote_df(df_1_day)
+    anony_vote_hist_1_day = anony_vote_df_1_day["type"].value_counts()
+    # anony_vote_df_1_hour = get_anony_vote_df(df_1_hour)
+    # anony_vote_hist_1_hour = anony_vote_df_1_hour["type"].value_counts()
+    anony_vote_hist = merge_counts(
+        [anony_vote_hist_all, anony_vote_hist_1_day],
+        on="type",
+        names=["All", "Last Day"],
+    )
+    anony_vote_hist_md = anony_vote_hist.to_markdown(index=False, tablefmt="github")
+
+    # Last 24 hours
+    chat_1_day = df_1_day[df_1_day["type"] == "chat"]
+    num_chats_last_24_hours = []
+    base = df_1_day["tstamp"].min()
+    for i in range(24, 0, -1):
+        left = base + (i - 1) * 3600
+        right = base + i * 3600
+        num = ((chat_1_day["tstamp"] >= left) & (chat_1_day["tstamp"] < right)).sum()
+        num_chats_last_24_hours.append(num)
+    times = [
+        datetime.datetime.fromtimestamp(
+            base + i * 3600, tz=timezone("US/Pacific")
+        ).strftime("%Y-%m-%d %H:%M:%S %Z")
+        for i in range(24, 0, -1)
+    ]
+    last_24_hours_df = pd.DataFrame({"time": times, "value": num_chats_last_24_hours})
+    last_24_hours_md = last_24_hours_df.to_markdown(index=False, tablefmt="github")
+
+    # Last update datetime
+    last_updated_tstamp = now_t
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    # code.interact(local=locals())
+
+    return {
+        "chat_dates_bar": chat_dates_bar,
+        "model_hist_md": model_hist_md,
+        "action_hist_md": action_hist_md,
+        "anony_vote_hist_md": anony_vote_hist_md,
+        "num_chats_last_24_hours": last_24_hours_md,
+        "last_updated_datetime": last_updated_datetime,
+    }
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    basic_stats = report_basic_stats(log_files)
+
+    print(basic_stats["action_hist_md"] + "\n")
+    print(basic_stats["model_hist_md"] + "\n")
+    print(basic_stats["anony_vote_hist_md"] + "\n")
+    print(basic_stats["num_chats_last_24_hours"] + "\n")

arena_elo/elo_rating/clean_battle_data.py

@@ -0,0 +1,382 @@
+"""
+Clean chatbot arena battle log.
+
+Usage:
+python3 clean_battle_data.py --mode conv_release
+"""
+import argparse
+import datetime
+import json
+import os
+import sys
+from pytz import timezone
+import time
+import PIL
+from PIL import ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+from tqdm import tqdm
+
+from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
+from .utils import detect_language, get_time_stamp_from_date
+
+VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote"]
+IDENTITY_WORDS = [
+    "vicuna",
+    "lmsys",
+    "koala",
+    "uc berkeley",
+    "open assistant",
+    "laion",
+    "chatglm",
+    "chatgpt",
+    "gpt-4",
+    "openai",
+    "anthropic",
+    "claude",
+    "bard",
+    "palm",
+    "lamda",
+    "google",
+    "llama",
+    "qianwan",
+    "alibaba",
+    "mistral",
+    "zhipu",
+    "KEG lab",
+    "01.AI",
+    "AI2",
+    "Tülu",
+    "Tulu",
+    "NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.",
+    "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES.",
+    "API REQUEST ERROR. Please increase the number of max tokens.",
+    "**API REQUEST ERROR** Reason: The response was blocked.",
+    "**API REQUEST ERROR**",
+]
+
+for i in range(len(IDENTITY_WORDS)):
+    IDENTITY_WORDS[i] = IDENTITY_WORDS[i].lower()
+
+
+def remove_html(raw):
+    if raw.startswith("<h3>"):
+        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
+    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
+        return raw[13:]
+    return raw
+
+
+def to_openai_format(messages):
+    roles = ["user", "assistant"]
+    ret = []
+    for i, x in enumerate(messages):
+        ret.append({"role": roles[i % 2], "content": x[1]})
+    return ret
+
+
+def replace_model_name(old_name, tstamp):
+    replace_dict = {
+        "bard": "palm-2",
+        "claude-v1": "claude-1",
+        "claude-instant-v1": "claude-instant-1",
+        "oasst-sft-1-pythia-12b": "oasst-pythia-12b",
+        "claude-2": "claude-2.0",
+        "PlayGroundV2": "Playground v2",
+    }
+    if old_name in ["gpt-4", "gpt-3.5-turbo"]:
+        if tstamp > 1687849200:
+            return old_name + "-0613"
+        else:
+            return old_name + "-0314"
+    if old_name in replace_dict:
+        return replace_dict[old_name]
+    return old_name
+
+
+def read_file(filename):
+    data = []
+    for retry in range(5):
+        try:
+            # lines = open(filename).readlines()
+            for l in open(filename):
+                row = json.loads(l)
+                if row["type"] in VOTES:
+                    data.append(row)
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+    return data
+
+
+def read_file_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+def load_image(image_path):
+    try:
+        return PIL.Image.open(image_path)
+    except:
+        return None
+
+def clean_battle_data(
+    log_files, exclude_model_names, ban_ip_list=None, sanitize_ip=False, mode="simple", task_name="t2s"
+):
+    data = read_file_parallel(log_files, num_threads=16)
+
+    convert_type = {
+        "leftvote": "model_a",
+        "rightvote": "model_b",
+        "tievote": "tie",
+        "bothbad_vote": "tie (bothbad)",
+    }
+    
+    all_models = set()
+    all_ips = dict()
+    ct_anony = 0
+    ct_invalid = 0
+    ct_leaked_identity = 0
+    ct_banned = 0
+    battles = []
+    for row in tqdm(data, desc="Cleaning"):
+        if row["models"][0] is None or row["models"][1] is None:
+            continue
+
+        # Resolve model names
+        models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+        if "model_name" in row["states"][0]:
+            models_hidden = [
+                row["states"][0]["model_name"],
+                row["states"][1]["model_name"],
+            ]
+            if models_hidden[0] is None:
+                models_hidden = models_public
+        else:
+            models_hidden = models_public
+
+        if (models_public[0] == "" and models_public[1] != "") or (
+            models_public[1] == "" and models_public[0] != ""
+        ):
+            ct_invalid += 1
+            continue
+
+        if models_public[0] == "" or models_public[0] == "Model A":
+            anony = True
+            models = models_hidden
+            ct_anony += 1
+        else:
+            anony = False
+            models = models_public
+            if not models_public == models_hidden:
+                ct_invalid += 1
+                continue
+
+        # # Detect langauge
+        # state = row["states"][0]
+        # if state["offset"] >= len(state["messages"]):
+        #     ct_invalid += 1
+        #     continue
+        # lang_code = detect_language(state["messages"][state["offset"]][1])
+
+        # # Drop conversations if the model names are leaked
+        # leaked_identity = False
+        # messages = ""
+        # for i in range(2):
+        #     state = row["states"][i]
+        #     for turn_idx, (role, msg) in enumerate(
+        #         state["messages"][state["offset"] :]
+        #     ):
+        #         if msg:
+        #             messages += msg.lower()
+        # for word in IDENTITY_WORDS:
+        #     if word in messages:
+        #         leaked_identity = True
+        #         break
+
+        # if leaked_identity:
+        #     ct_leaked_identity += 1
+        #     continue
+
+        # Replace bard with palm
+        if task_name == "image_editing":
+            if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
+                # print(f"Invalid model names: {models}")
+                ct_invalid += 1
+                continue
+            models = [x[len("imagenhub_"):-len("_edition")] for x in models]
+        elif task_name == "t2i_generation":
+            if not all("playground" in x.lower() or (x.startswith("imagenhub_") and x.endswith("_generation")) for x in models):
+                # print(f"Invalid model names: {models}")
+                ct_invalid += 1
+                continue
+            # models = [x[len("imagenhub_"):-len("_generation")] for x in models]
+            for i, model_name in enumerate(models):
+                if model_name.startswith("imagenhub_"):
+                    models[i] = model_name[len("imagenhub_"):-len("_generation")]
+
+        else:
+            raise ValueError(f"Invalid task_name: {task_name}")
+        models = [replace_model_name(m, row["tstamp"]) for m in models]
+        
+        # Exclude certain models
+        if exclude_model_names and any(x in exclude_model_names for x in models):
+            ct_invalid += 1
+            continue
+        
+        # if models[0] not in model_infos or models[1] not in model_infos:
+        #     continue
+
+        # # Exclude votes before the starting date
+        # if model_infos and (model_infos[models[0]]["starting_from"] > row["tstamp"] or model_infos[models[1]]["starting_from"] > row["tstamp"]):
+        #     print(f"Invalid vote before the valid starting date for {models[0]} and {models[1]}")
+        #     ct_invalid += 1
+        #     continue
+        
+        
+
+        if mode == "conv_release":
+            # assert the two images are the same
+            date = datetime.datetime.fromtimestamp(row["tstamp"], tz=timezone("US/Pacific")).strftime("%Y-%m-%d") # 2024-02-29
+            image_path_format = f"{LOG_ROOT_DIR}/{date}-convinput_images/input_image_"
+            image_path_0 = image_path_format + str(row["states"][0]["conv_id"]) + ".png"
+            image_path_1 = image_path_format + str(row["states"][1]["conv_id"]) + ".png"
+            if not os.path.exists(image_path_0) or not os.path.exists(image_path_1):
+                print(f"Image not found for {image_path_0} or {image_path_1}")
+                ct_invalid += 1
+                continue
+            
+            image_0 = load_image(image_path_0)
+            image_1 = load_image(image_path_1)
+            if image_0 is None or image_1 is None:
+                print(f"Image not found for {image_path_0} or {image_path_1}")
+                ct_invalid += 1
+                continue
+            if image_0.tobytes() != image_1.tobytes():
+                print(f"Image not the same for {image_path_0} and {image_path_1}")
+                ct_invalid += 1
+                continue
+        
+        
+        question_id = row["states"][0]["conv_id"]
+        # conversation_a = to_openai_format(
+        #     row["states"][0]["messages"][row["states"][0]["offset"] :]
+        # )
+        # conversation_b = to_openai_format(
+        #     row["states"][1]["messages"][row["states"][1]["offset"] :]
+        # )
+
+        ip = row["ip"]
+        if ip not in all_ips:
+            all_ips[ip] = {"ip": ip, "count": 0, "sanitized_id": len(all_ips)}
+        all_ips[ip]["count"] += 1
+        if sanitize_ip:
+            user_id = f"arena_user_{all_ips[ip]['sanitized_id']}"
+        else:
+            user_id = f"{all_ips[ip]['ip']}"
+
+        if ban_ip_list is not None and ip in ban_ip_list:
+            ct_banned += 1
+            continue
+
+        # Save the results
+        battles.append(
+            dict(
+                question_id=question_id,
+                model_a=models[0],
+                model_b=models[1],
+                winner=convert_type[row["type"]],
+                judge=f"arena_user_{user_id}",
+                # conversation_a=conversation_a,
+                # conversation_b=conversation_b,
+                # turn=len(conversation_a) // 2,
+                anony=anony,
+                # language=lang_code,
+                tstamp=row["tstamp"],
+            )
+        )
+
+        all_models.update(models_hidden)
+    battles.sort(key=lambda x: x["tstamp"])
+    last_updated_tstamp = battles[-1]["tstamp"]
+
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    print(
+        f"#votes: {len(data)}, #invalid votes: {ct_invalid}, "
+        f"#leaked_identity: {ct_leaked_identity} "
+        f"#banned: {ct_banned} "
+    )
+    print(f"#battles: {len(battles)}, #anony: {ct_anony}")
+    print(f"#models: {len(all_models)}, {all_models}")
+    print(f"last-updated: {last_updated_datetime}")
+
+    if ban_ip_list is not None:
+        for ban_ip in ban_ip_list:
+            if ban_ip in all_ips:
+                del all_ips[ban_ip]
+    print("Top 30 IPs:")
+    print(sorted(all_ips.values(), key=lambda x: x["count"], reverse=True)[:30])
+    return battles
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    parser.add_argument(
+        "--mode", type=str, choices=["simple", "conv_release"], default="simple"
+    )
+    parser.add_argument("--task_name", type=str, choices=["t2s", "i2s"])
+    parser.add_argument("--exclude-model-names", type=str, nargs="+")
+    parser.add_argument("--ban-ip-file", type=str)
+    parser.add_argument("--sanitize-ip", action="store_true", default=False)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    ban_ip_list = json.load(open(args.ban_ip_file)) if args.ban_ip_file else None
+
+    battles = clean_battle_data(
+        log_files, args.exclude_model_names or [], ban_ip_list, args.sanitize_ip, args.mode, args.task_name
+    )
+    last_updated_tstamp = battles[-1]["tstamp"]
+    cutoff_date = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y%m%d")
+
+    if args.mode == "simple":
+        for x in battles:
+            for key in [
+                "conversation_a",
+                "conversation_b",
+                "question_id",
+            ]:
+                if key in x:
+                    del x[key]
+        print("Samples:")
+        for i in range(min(4, len(battles))):
+            print(battles[i])
+        output = f"clean_battle_{args.task_name}_{cutoff_date}.json"
+    elif args.mode == "conv_release":
+        # new_battles = []
+        # for x in battles:
+        #     if not x["anony"]:
+        #         continue
+        #     for key in []:
+        #         del x[key]
+        #     new_battles.append(x)
+        # battles = new_battles
+        output = f"clean_battle_{args.task_name}_conv_{cutoff_date}.json"
+
+    with open(output, "w") as fout:
+        json.dump(battles, fout, indent=2, ensure_ascii=False)
+    print(f"Write cleaned data to {output}")
+    
+    with open("cut_off_date.txt", "w") as fout:
+        fout.write(cutoff_date)

arena_elo/elo_rating/elo_analysis.py

@@ -0,0 +1,378 @@
+import argparse
+from collections import defaultdict
+import datetime
+import json
+import math
+import pickle
+from pytz import timezone
+
+import numpy as np
+import pandas as pd
+import plotly.express as px
+from tqdm import tqdm
+
+from .model_registry import get_model_info
+from .basic_stats import get_log_files
+from .clean_battle_data import clean_battle_data
+        
+pd.options.display.float_format = "{:.2f}".format
+
+
+def compute_elo(battles, K=4, SCALE=400, BASE=10, INIT_RATING=1000):
+    rating = defaultdict(lambda: INIT_RATING)
+
+    for rd, model_a, model_b, winner in battles[
+        ["model_a", "model_b", "winner"]
+    ].itertuples():
+        ra = rating[model_a]
+        rb = rating[model_b]
+        ea = 1 / (1 + BASE ** ((rb - ra) / SCALE))
+        eb = 1 / (1 + BASE ** ((ra - rb) / SCALE))
+        if winner == "model_a":
+            sa = 1
+        elif winner == "model_b":
+            sa = 0
+        elif winner == "tie" or winner == "tie (bothbad)":
+            sa = 0.5
+        else:
+            raise Exception(f"unexpected vote {winner}")
+        rating[model_a] += K * (sa - ea)
+        rating[model_b] += K * (1 - sa - eb)
+
+    return dict(rating)
+
+
+def get_bootstrap_result(battles, func_compute_elo, num_round=1000):
+    rows = []
+    for i in tqdm(range(num_round), desc="bootstrap"):
+        tmp_battles = battles.sample(frac=1.0, replace=True)
+        rows.append(func_compute_elo(tmp_battles))
+    df = pd.DataFrame(rows)
+    return df[df.median().sort_values(ascending=False).index]
+
+
+def compute_elo_mle_with_tie(df, SCALE=400, BASE=10, INIT_RATING=1000):
+    from sklearn.linear_model import LogisticRegression
+
+    models = pd.concat([df["model_a"], df["model_b"]]).unique()
+    models = pd.Series(np.arange(len(models)), index=models)
+
+    # duplicate battles
+    df = pd.concat([df, df], ignore_index=True)
+    p = len(models.index)
+    n = df.shape[0]
+
+    X = np.zeros([n, p])
+    X[np.arange(n), models[df["model_a"]]] = +math.log(BASE)
+    X[np.arange(n), models[df["model_b"]]] = -math.log(BASE)
+
+    # one A win => two A win
+    Y = np.zeros(n)
+    Y[df["winner"] == "model_a"] = 1.0
+
+    # one tie => one A win + one B win
+    # find tie + tie (both bad) index
+    tie_idx = (df["winner"] == "tie") | (df["winner"] == "tie (bothbad)")
+    tie_idx[len(tie_idx) // 2 :] = False
+    Y[tie_idx] = 1.0
+
+    lr = LogisticRegression(fit_intercept=False)
+    lr.fit(X, Y)
+
+    elo_scores = SCALE * lr.coef_[0] + INIT_RATING
+    # calibrate llama-13b to 800 if applicable
+    if "llama-13b" in models.index:
+        elo_scores += 800 - elo_scores[models["llama-13b"]]
+    return pd.Series(elo_scores, index=models.index).sort_values(ascending=False)
+
+
+def get_median_elo_from_bootstrap(bootstrap_df):
+    median = dict(bootstrap_df.quantile(0.5))
+    median = {k: int(v + 0.5) for k, v in median.items()}
+    return median
+
+
+def compute_pairwise_win_fraction(battles, model_order, limit_show_number=None):
+    # Times each model wins as Model A
+    a_win_ptbl = pd.pivot_table(
+        battles[battles["winner"] == "model_a"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting times each model wins as Model B
+    b_win_ptbl = pd.pivot_table(
+        battles[battles["winner"] == "model_b"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting number of A-B pairs
+    num_battles_ptbl = pd.pivot_table(
+        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
+    )
+
+    # Computing the proportion of wins for each model as A and as B
+    # against all other models
+    row_beats_col_freq = (a_win_ptbl + b_win_ptbl.T) / (
+        num_battles_ptbl + num_battles_ptbl.T
+    )
+
+    if model_order is None:
+        prop_wins = row_beats_col_freq.mean(axis=1).sort_values(ascending=False)
+        model_order = list(prop_wins.keys())
+
+    if limit_show_number is not None:
+        model_order = model_order[:limit_show_number]
+
+    # Arrange ordering according to proprition of wins
+    row_beats_col = row_beats_col_freq.loc[model_order, model_order]
+    return row_beats_col
+
+
+def visualize_leaderboard_table(rating):
+    models = list(rating.keys())
+    models.sort(key=lambda k: -rating[k])
+
+    emoji_dict = {
+        1: "🥇",
+        2: "🥈",
+        3: "🥉",
+    }
+
+    md = ""
+    md += "| Rank | Model | Elo Rating | Description |\n"
+    md += "| --- | --- | --- | --- |\n"
+    for i, model in enumerate(models):
+        rank = i + 1
+        minfo = get_model_info(model)
+        emoji = emoji_dict.get(rank, "")
+        md += f"| {rank} | {emoji} [{model}]({minfo.link}) | {rating[model]:.0f} | {minfo.description} |\n"
+
+    return md
+
+
+def visualize_pairwise_win_fraction(battles, model_order):
+    row_beats_col = compute_pairwise_win_fraction(battles, model_order)
+    fig = px.imshow(
+        row_beats_col,
+        color_continuous_scale="RdBu",
+        text_auto=".2f",
+        height=700,
+        width=700,
+    )
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(
+        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Fraction of A Wins: %{z}<extra></extra>"
+    )
+
+    return fig
+
+
+def visualize_battle_count(battles, model_order):
+    ptbl = pd.pivot_table(
+        battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0
+    )
+    battle_counts = ptbl + ptbl.T
+    fig = px.imshow(
+        battle_counts.loc[model_order, model_order],
+        text_auto=True,
+        height=700,
+        width=700,
+    )
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(
+        hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Count: %{z}<extra></extra>"
+    )
+    return fig
+
+
+def visualize_average_win_rate(battles, limit_show_number):
+    row_beats_col_freq = compute_pairwise_win_fraction(
+        battles, None, limit_show_number=limit_show_number
+    )
+    fig = px.bar(
+        row_beats_col_freq.mean(axis=1).sort_values(ascending=False),
+        text_auto=".2f",
+        height=500,
+        width=700,
+    )
+    fig.update_layout(
+        yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False
+    )
+    return fig
+
+
+def visualize_bootstrap_elo_rating(df, df_final, limit_show_number):
+    bars = (
+        pd.DataFrame(
+            dict(
+                lower=df.quantile(0.025),
+                rating=df_final,
+                upper=df.quantile(0.975),
+            )
+        )
+        .reset_index(names="model")
+        .sort_values("rating", ascending=False)
+    )
+    bars = bars[:limit_show_number]
+    bars["error_y"] = bars["upper"] - bars["rating"]
+    bars["error_y_minus"] = bars["rating"] - bars["lower"]
+    bars["rating_rounded"] = np.round(bars["rating"], 2)
+    fig = px.scatter(
+        bars,
+        x="model",
+        y="rating",
+        error_y="error_y",
+        error_y_minus="error_y_minus",
+        text="rating_rounded",
+        height=500,
+        width=700,
+    )
+    fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
+    return fig
+
+
+def report_elo_analysis_results(battles_json, rating_system="bt", num_bootstrap=100, anony_only=True):
+    battles = pd.DataFrame(battles_json)
+    battles = battles.sort_values(ascending=True, by=["tstamp"])
+    # Only use anonymous votes
+    if anony_only:
+        battles = battles[battles["anony"]].reset_index(drop=True)
+    battles_no_ties = battles[~battles["winner"].str.contains("tie")]
+
+    # Online update
+    elo_rating_online = compute_elo(battles)
+
+    if rating_system == "bt":
+        bootstrap_df = get_bootstrap_result(
+            battles, compute_elo_mle_with_tie, num_round=num_bootstrap
+        )
+        elo_rating_final = compute_elo_mle_with_tie(battles)
+    elif rating_system == "elo":
+        bootstrap_df = get_bootstrap_result(
+            battles, compute_elo, num_round=num_bootstrap
+        )
+        elo_rating_median = get_median_elo_from_bootstrap(bootstrap_df)
+        elo_rating_final = elo_rating_median
+
+    model_order = list(elo_rating_final.keys())
+    model_order.sort(key=lambda k: -elo_rating_final[k])
+
+    limit_show_number = 25  # limit show number to make plots smaller
+    model_order = model_order[:limit_show_number]
+
+    # leaderboard_table_df: elo rating, variance, 95% interval, number of battles
+    leaderboard_table_df = pd.DataFrame(
+        {
+            "rating": elo_rating_final,
+            "variance": bootstrap_df.var(),
+            "rating_q975": bootstrap_df.quantile(0.975),
+            "rating_q025": bootstrap_df.quantile(0.025),
+            "num_battles": battles["model_a"].value_counts()
+            + battles["model_b"].value_counts(),
+        }
+    )
+
+    # Plots
+    leaderboard_table = visualize_leaderboard_table(elo_rating_final)
+    win_fraction_heatmap = visualize_pairwise_win_fraction(battles_no_ties, model_order)
+    battle_count_heatmap = visualize_battle_count(battles_no_ties, model_order)
+    average_win_rate_bar = visualize_average_win_rate(
+        battles_no_ties, limit_show_number
+    )
+    bootstrap_elo_rating = visualize_bootstrap_elo_rating(
+        bootstrap_df, elo_rating_final, limit_show_number
+    )
+
+    last_updated_tstamp = battles["tstamp"].max()
+    last_updated_datetime = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y-%m-%d %H:%M:%S %Z")
+
+    return {
+        "rating_system": rating_system,
+        "elo_rating_online": elo_rating_online,
+        "elo_rating_final": elo_rating_final,
+        "leaderboard_table": leaderboard_table,
+        "win_fraction_heatmap": win_fraction_heatmap,
+        "battle_count_heatmap": battle_count_heatmap,
+        "average_win_rate_bar": average_win_rate_bar,
+        "bootstrap_elo_rating": bootstrap_elo_rating,
+        "last_updated_datetime": last_updated_datetime,
+        "last_updated_tstamp": last_updated_tstamp,
+        "bootstrap_df": bootstrap_df,
+        "leaderboard_table_df": leaderboard_table_df,
+    }
+
+
+def pretty_print_elo_rating(rating):
+    model_order = list(rating.keys())
+    model_order.sort(key=lambda k: -rating[k])
+    for i, model in enumerate(model_order):
+        print(f"{i+1:2d}, {model:25s}, {rating[model]:.0f}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--clean-battle-file", type=str)
+    parser.add_argument("--max-num-files", type=int)
+    parser.add_argument("--num-bootstrap", type=int, default=100)
+    parser.add_argument(
+        "--rating-system", type=str, choices=["bt", "elo"], default="bt"
+    )
+    parser.add_argument("--exclude-tie", action="store_true", default=False)
+    args = parser.parse_args()
+
+    np.random.seed(42)
+
+    if args.clean_battle_file:
+        # Read data from a cleaned battle files
+        battles = pd.read_json(args.clean_battle_file)
+    else:
+        # Read data from all log files
+        log_files = get_log_files(args.max_num_files)
+        battles = clean_battle_data(log_files)
+
+    anony_results = report_elo_analysis_results(
+        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=True
+    )
+    full_results = report_elo_analysis_results(
+        battles, rating_system=args.rating_system, num_bootstrap=args.num_bootstrap, anony_only=False
+    )
+    
+
+    print("# Online Elo")
+    pretty_print_elo_rating(anony_results["elo_rating_online"])
+    print("# Median")
+    pretty_print_elo_rating(anony_results["elo_rating_final"])
+    print(f"last update : {anony_results['last_updated_datetime']}")
+
+    last_updated_tstamp = full_results["last_updated_tstamp"]
+    cutoff_date = datetime.datetime.fromtimestamp(
+        last_updated_tstamp, tz=timezone("US/Pacific")
+    ).strftime("%Y%m%d")
+
+
+    results = {
+        "anony": anony_results,
+        "full": full_results,
+    }
+    with open(f"elo_results_{cutoff_date}.pkl", "wb") as fout:
+        pickle.dump(results, fout)

arena_elo/elo_rating/generate_leaderboard.py

@@ -0,0 +1,71 @@
+import fire
+import json
+import pandas as pd
+import pickle
+
+
+def main(
+        model_info_file: str,
+        elo_rating_pkl: str,
+        output_csv: str
+):
+    model_info = json.load(open(model_info_file))
+
+    with open(elo_rating_pkl, "rb") as fin:
+        elo_rating_results = pickle.load(fin)
+
+    anony_elo_rating_results = elo_rating_results["anony"]
+    full_elo_rating_results = elo_rating_results["full"]
+    anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
+    full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
+
+    # Model,MT-bench (score),Arena Elo rating,MMLU,License,Link
+    fields = ["key", "Model", "Arena Elo rating (anony)", "Arena Elo rating (full)", "License", "Organization", "Link"]
+    # set Organization and license to empty for now
+    all_models = anony_leaderboard_data.index.tolist()
+
+    for model in all_models:
+        if not model in model_info:
+            model_info[model] = {}
+            model_info[model]["License"] = "N/A"
+            model_info[model]["Organization"] = "N/A"
+            model_info[model]["Link"] = "N/A"
+        model_info[model]["Model"] = model
+        model_info[model]["key"] = model
+
+        if model in anony_leaderboard_data.index:
+            model_info[model]["Arena Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]
+        else:
+            model_info[model]["Arena Elo rating (anony)"] = 0
+
+        if model in full_elo_rating_results["leaderboard_table_df"].index:
+            model_info[model]["Arena Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
+        else:
+            model_info[model]["Arena Elo rating (full)"] = 0
+        # if model in anony_leaderboard_data.index:
+        #     model_info[model]["Arena Elo rating"] = anony_leaderboard_data.loc[model, "rating"]
+        # else:
+        #     model_info[model]["Arena Elo rating"] = 0
+
+    final_model_info = {}
+    for model in model_info:
+        if "Model" in model_info[model]:
+            final_model_info[model] = model_info[model]
+    model_info = final_model_info
+
+    exclude_keys = ['starting_from']
+    for key in exclude_keys:
+        for model in model_info:
+            if key in model_info[model]:
+                del model_info[model][key]
+    df = pd.DataFrame(model_info).T
+    df = df[fields]
+    # sort by anony rating
+    df = df.sort_values(by=["Arena Elo rating (anony)"], ascending=False)
+    df.to_csv(output_csv, index=False)
+    print("Leaderboard data saved to", output_csv)
+    print(df)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/inspect_conv_rating.py

@@ -0,0 +1,234 @@
+import argparse
+import code
+import datetime
+import json
+import os
+from pytz import timezone
+import time
+
+import pandas as pd
+from tqdm import tqdm
+import csv
+
+import base64
+from icecream import ic
+from openai import OpenAI
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+def get_log_files(max_num_files=None):
+    dates = []
+    for month in [2, 3]:
+        for day in range(1, 32):
+            dates.append(f"2024-{month:02d}-{day:02d}")
+
+    num_servers = 1
+    filenames = []
+    for d in dates:
+        for i in range(num_servers):
+            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
+            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
+            if os.path.exists(name):
+                filenames.append(name)
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def pretty_print_conversation(messages):
+    for role, msg in messages:
+        print(f"[[{role}]]: {msg}")
+
+
+def get_gpt4v_response(client, img_bs64=None, text_prompt="", use_vision=False):
+    if use_vision:
+        response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": text_prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{img_bs64}"
+                        }
+                    },
+                ],
+            }
+        ],
+        max_tokens=100,
+        )
+    else:
+        response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": text_prompt},
+                ],
+            }
+        ],
+        max_tokens=100,
+        )
+    return response.choices[0].message.content
+
+task_template_map = {
+    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
+    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
+    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
+    "pair_rate_wexplanation": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
+    "pair_rate": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
+}
+
+def inspect_convs(log_files):
+    ic(log_files)
+    data = []
+    total_vote = 0
+    correct_vote = 0
+        
+    client = OpenAI()
+    with open('all_pairvote_log_wgpt_prtchatbot.csv', 'w', newline='') as csvfile:
+        # fieldnames = ['tstamp', 'type', 'model_1', 'model_2', 'template_name_1', 'template_name_2', 'system_message_1', 'system_message_2', 'role_1', 'role_2', 'instruction_1', 'instruction_2', 'message_1', 'message_2', 'offset_1', 'offset_2', 'conv_id_1', 'conv_id_2', 'model_name_1', 'model_name_2', 'ip']
+        fieldnames = ['tstamp', 'type', 'models', 'states', 'ip', 'gpt_vote']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        
+        # Write the header
+        writer.writeheader()
+
+        for filename in tqdm(log_files, desc="read files"):
+            for retry in range(5):
+                try:
+                    lines = open(filename).readlines()
+                    break
+                except FileNotFoundError:
+                    time.sleep(2)
+            
+            for l in lines:
+                row = json.loads(l)
+                
+                if "states" not in row:
+                    continue
+                if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                    continue
+                
+                model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
+                
+
+                # Iterate through each state and write the relevant information
+                if not len(row["states"][0]['messages']): continue
+                # ic(row["states"][0]['messages'][1][1])
+
+                if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
+                total_vote += 1
+                # row = {
+                #     'tstamp': row['tstamp'],
+                #     'type': row['type'],
+                #     'model_1': row['models'][0],
+                #     'model_2': row['models'][1],
+                #     'template_name_1': row["states"][0]['template_name'],
+                #     'system_message_1': row["states"][0]['system_message'],
+                #     'template_name_2': row["states"][1]['template_name'],
+                #     'system_message_2': row["states"][1]['system_message'],
+                #     'role_1': row["states"][0]['roles'],
+                #     'role_2': row["states"][1]['roles'],
+                #     'instruction_1': row["states"][0]['messages'][0][1],
+                #     'instruction_2': row["states"][1]['messages'][0][1],
+                #     'message_1': row["states"][0]['messages'][1][1],
+                #     'message_2': row["states"][1]['messages'][1][1],
+                #     'offset_1': row["states"][0]['offset'],
+                #     'offset_2': row["states"][1]['offset'],
+                #     'conv_id_1': row["states"][0]['conv_id'],
+                #     'conv_id_2': row["states"][1]['conv_id'],
+                #     'model_name_1': row["states"][0]['model_name'],
+                #     'model_name_2': row["states"][1]['model_name'],
+                #     'ip': row['ip']
+                # }
+                # writer.writerow(row)
+                # Convert complex objects to JSON strings
+                # TODO: check two image are the same
+                conv_id = row["states"][0]['conv_id']
+                image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
+                if not os.path.exists(image_path):
+                    response = "NA"
+                    ic(image_path)
+                else:
+                    base64_image = encode_image(image_path)
+                    left_response = row["states"][0]['messages'][1][1]
+                    right_response = row["states"][1]['messages'][1][1]
+                    sep = "-" * 20
+                    instruction = row["states"][0]['messages'][0][1]
+                    generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
+                    text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
+                    # ic(text_prompt)
+                    try:
+                        response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
+                    except:
+                        ic(">>> skip")
+                        response = "NA"
+                    
+                    # response = get_gpt4v_response(client, img_bs64=base64_image, text_prompt=text_prompt, use_vision=True)
+                    ic(row['type'], response)
+                    if response.strip() not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                        response = "NA"
+                    # ic(generated_sentence)
+                    
+                    # if row['type'] == "leftvote":
+                    #     row['type'] = "A"
+                    # elif row['type'] == "rightvote":
+                    #     row['type'] = "B"
+                    # elif row['type'] in ["bothbad_vote", "tievote"]:
+                    #     row['type'] = "C"
+                    if row['type'] == response.strip():
+                        correct_vote += 1
+                row['models'] = json.dumps(row['models'])
+                row['states'] = json.dumps(row['states'], ensure_ascii=False)
+                row['gpt_vote'] = response
+                
+                # Write the modified row to the CSV file
+                writer.writerow(row)
+                # if row["type"] == "leftvote":
+                #     winner, loser = model_names[0], model_names[1]
+                #     winner_conv, loser_conv = row["states"][0], row["states"][1]
+                # elif row["type"] == "rightvote":
+                #     loser, winner = model_names[0], model_names[1]
+                #     loser_conv, winner_conv = row["states"][0], row["states"][1]
+
+                # if loser == "llava-v1.5-13b" and winner == "llava-v1.5-13b":
+                #     print("=" * 20)
+                #     print(f"Winner: {winner}")
+                #     pretty_print_conversation(winner_conv["messages"])
+                #     print(f"Loser: {loser}")
+                #     pretty_print_conversation(loser_conv["messages"])
+                #     print("=" * 20)
+                #     input()
+                # if row['type'] == 'bothbad_vote':
+                #     from icecream import ic
+                #     ic(model_names)
+                # if row["type"] == "bothbad_vote" and "gpt-4-vision-preview" in model_names:
+                #    print("=" * 20)
+                #    print(f"Model A: {model_names[0]}")
+                #    pretty_print_conversation(row["states"][0]["messages"])
+                #    print(f"Model B: {model_names[1]}")
+                #    pretty_print_conversation(row["states"][1]["messages"])
+                #    print("=" * 20)
+                #    input()
+                # if correct_vote >= 300: break
+    ic(total_vote, correct_vote)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    
+
+            
+    inspect_convs(log_files)

arena_elo/elo_rating/inspect_cost.py

@@ -0,0 +1,177 @@
+import fire
+import time
+import json
+from collections import defaultdict
+from .basic_stats import get_log_files, NUM_SERVERS, LOG_ROOT_DIR
+from .utils import detect_language, get_time_stamp_from_date, get_input_image_path, load_image_from_path
+from tqdm import tqdm
+VOTES = ["tievote", "leftvote", "rightvote", "bothbad_vote", "chat"]
+
+
+def remove_html(raw):
+    if raw.startswith("<h3>"):
+        return raw[raw.find(": ") + 2 : -len("</h3>\n")]
+    if raw.startswith("### Model A: ") or raw.startswith("### Model B: "):
+        return raw[13:]
+    return raw
+
+
+def read_file(filename):
+    data = []
+    for retry in range(5):
+        try:
+            # lines = open(filename).readlines()
+            for l in open(filename):
+                row = json.loads(l)
+                if row["type"] in VOTES:
+                    data.append(row)
+            break
+        except FileNotFoundError:
+            time.sleep(2)
+    return data
+
+
+def read_file_parallel(log_files, num_threads=16):
+    data_all = []
+    from multiprocessing import Pool
+
+    with Pool(num_threads) as p:
+        ret_all = list(tqdm(p.imap(read_file, log_files), total=len(log_files)))
+        for ret in ret_all:
+            data_all.extend(ret)
+    return data_all
+
+def num_tokens(s:str):
+    if s is None:
+        return 0
+    return len(s) / 4
+
+def main(
+):
+    log_files = get_log_files()
+    data = read_file_parallel(log_files)
+    
+    all_model_counts = defaultdict(int)
+    all_model_input_tokens_counts = defaultdict(list)
+    all_model_output_tokens_counts = defaultdict(list)
+    all_model_image_sizes = defaultdict(list)
+    chat_battle_counts = defaultdict(int)
+    for row in tqdm(data, desc="counting"):
+        if row['type'] == "chat":
+            chat_battle_counts["chat"] += 1
+            all_model_counts[row['model']] += 1
+            tstamp = row["tstamp"]
+            conv_id = row["state"]["conv_id"]
+            
+            image = load_image_from_path(get_input_image_path(tstamp, conv_id))
+            if image is None:
+                image_size = None
+            else:
+                image_size = load_image_from_path(get_input_image_path(tstamp, conv_id)).size
+            all_model_image_sizes[row['model']].append(image_size)
+            try:
+                for message in row["state"]["messages"][row["state"]["offset"] :: 2]:
+                    all_model_input_tokens_counts[row['model']].append(num_tokens(message[1]))
+                for message in row["state"]["messages"][row["state"]["offset"] + 1 :: 2]:
+                    all_model_output_tokens_counts[row['model']].append(num_tokens(message[1]))
+            except Exception as e:
+                print(row)
+                raise e
+            
+        else:
+            chat_battle_counts[row['type']] += 1
+            if row["models"][0] is None or row["models"][1] is None:
+                continue
+
+            # Resolve model names
+            models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+            if "model_name" in row["states"][0]:
+                models_hidden = [
+                    row["states"][0]["model_name"],
+                    row["states"][1]["model_name"],
+                ]
+                if models_hidden[0] is None:
+                    models_hidden = models_public
+            else:
+                models_hidden = models_public
+
+            if (models_public[0] == "" and models_public[1] != "") or (
+                models_public[1] == "" and models_public[0] != ""
+            ):
+                continue
+
+            if models_public[0] == "" or models_public[0] == "Model A":
+                anony = True
+                models = models_hidden
+            else:
+                anony = False
+                models = models_public
+                if not models_public == models_hidden:
+                    continue
+            
+            all_model_counts[models[0]] += 1
+            all_model_counts[models[1]] += 1
+            tstamp = row["tstamp"]
+            conv_id1 = row["states"][0]["conv_id"]
+            conv_id2 = row["states"][1]["conv_id"]
+            
+            image1 = load_image_from_path(get_input_image_path(tstamp, conv_id1))
+            image2 = load_image_from_path(get_input_image_path(tstamp, conv_id2))
+            all_model_image_sizes[models[0]].append(None if image1 is None else image1.size)
+            all_model_image_sizes[models[1]].append(None if image2 is None else image2.size)
+            
+            for message in row["states"][0]["messages"][row["states"][0]["offset"] :: 2]:
+                all_model_input_tokens_counts[models[0]].append(num_tokens(message[1]))
+            for message in row["states"][0]["messages"][row["states"][0]["offset"] + 1 :: 2]:
+                all_model_output_tokens_counts[models[0]].append(num_tokens(message[1]))
+            for message in row["states"][1]["messages"][row["states"][1]["offset"] :: 2]:
+                all_model_input_tokens_counts[models[1]].append(num_tokens(message[1]))
+            for message in row["states"][1]["messages"][row["states"][1]["offset"] + 1 :: 2]:
+                all_model_output_tokens_counts[models[1]].append(num_tokens(message[1]))
+
+    print("### Chat battle counts (requests)")
+    print(json.dumps(chat_battle_counts, indent=4))
+    
+    print("### Model counts (requests)")
+    print(json.dumps(all_model_counts, indent=4))
+    
+    print("### Model Avg input tokens counts (tokens)")
+    average_input_tokens_counts = {}
+    for model, counts in all_model_input_tokens_counts.items():
+        average_input_tokens_counts[model] = sum(counts) / len(counts)
+    print(json.dumps(average_input_tokens_counts, indent=4))
+    
+    print("### Model AVg output tokens counts (tokens)")
+    average_output_tokens_counts = {}
+    for model, counts in all_model_output_tokens_counts.items():
+        average_output_tokens_counts[model] = sum(counts) / len(counts)
+    print(json.dumps(average_output_tokens_counts, indent=4))
+    
+    print("### Model Avg image sizes (height, width)")
+    average_image_sizes = {}
+    for model, sizes in all_model_image_sizes.items():
+        avg_height = sum([size[0] for size in sizes if size is not None]) / len(sizes)
+        avg_width = sum([size[1] for size in sizes if size is not None]) / len(sizes)
+        average_image_sizes[model] = (avg_height, avg_width)
+    print(json.dumps(average_image_sizes, indent=4))
+    
+    print("### GPT-4V estimated cost (USD)")
+    gpt_4v_name = "gpt-4-vision-preview"
+    gpt_4v_cost = {}
+    gpt_4v_cost['input'] = sum(all_model_input_tokens_counts[gpt_4v_name]) / 1000 * 0.01
+    gpt_4v_cost['output'] = sum(all_model_output_tokens_counts[gpt_4v_name]) / 1000 * 0.03
+    
+    all_image_cost = 0
+    for size in all_model_image_sizes[gpt_4v_name]:
+        if size is None:
+            continue
+        all_image_tokens = (size[0] // 512 + 1) * (size[1] // 512 + 1) * 170 + 85
+        all_image_cost += all_image_tokens / 1000 * 0.01
+    gpt_4v_cost['image'] = all_image_cost
+    print(json.dumps(gpt_4v_cost, indent=4))
+        
+    
+    
+    
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/inspect_elo_rating_pkl.py

@@ -0,0 +1,33 @@
+import pickle
+import plotly.graph_objects as go
+
+def output_figure(data, figure_name="battle_count_heatmap", label="annoy"):
+    fig = data[label][figure_name]
+    fig.update_layout(
+        height=700,
+        width=700,
+        title={'text': f'{figure_name}', 'x': 0.5, 'y': 0.07},
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        # coloraxis_colorscale=[[0.0, '#0d0887'], [1.0, '#f0f921']],
+        margin={'t': 60}
+    )
+    fig.write_image(f"{figure_name}.png")
+
+with open("./results/latest/elo_results.pkl",'rb') as f:
+    data = pickle.load(f)
+    print()
+    df = data["anony"]["leaderboard_table_df"]
+    # sort by rating 
+    print(data["anony"].keys())
+    
+    for figure_name in [ 'win_fraction_heatmap', 'battle_count_heatmap',]:
+        output_figure(data, figure_name, "anony")
+        
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df) 
+    df = data["full"]["leaderboard_table_df"]
+    # sort by rating 
+    df = df.sort_values(by=["rating"], ascending=False)
+    print(df)
+    print('done') 

arena_elo/elo_rating/model_registry.py

@@ -0,0 +1,578 @@
+"""Additional information of the models."""
+from collections import namedtuple, OrderedDict
+from typing import List
+
+
+ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description"])
+
+
+model_info = OrderedDict()
+
+
+def register_model_info(
+    full_names: List[str], simple_name: str, link: str, description: str
+):
+    info = ModelInfo(simple_name, link, description)
+
+    for full_name in full_names:
+        model_info[full_name] = info
+
+
+def get_model_info(name: str) -> ModelInfo:
+    if name in model_info:
+        return model_info[name]
+    else:
+        # To fix this, please use `register_model_info` to register your model
+        return ModelInfo(
+            name, "", "Register the description at arena.model/model_registry.py"
+        )
+
+
+register_model_info(
+    [
+        "IEITYuan/Yuan2-2B-Janus-hf",
+        "IEITYuan/Yuan2-2B-hf",
+        "IEITYuan/Yuan2-51B-hf",
+        "IEITYuan/Yuan2-102B-hf",
+    ],
+    "IEIT-Yuan2",
+    "https://github.com/IEIT-Yuan/Yuan-2.0",
+    "Yuan2.0 is a new generation Fundamental Large Language Model developed by IEIT System.",
+)
+
+register_model_info(
+    ["mixtral-8x7b-instruct-v0.1", "mistral-7b-instruct"],
+    "Mixtral of experts",
+    "https://mistral.ai/news/mixtral-of-experts/",
+    "A Mixture-of-Experts model by Mistral AI",
+)
+
+register_model_info(
+    ["gemini-pro"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["gemini-pro-vision"],
+    "Gemini",
+    "https://blog.google/technology/ai/google-gemini-pro-imagen-duet-ai-update/",
+    "Gemini by Google",
+)
+
+register_model_info(
+    ["solar-10.7b-instruct-v1.0"],
+    "SOLAR-10.7B-Instruct",
+    "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0",
+    "A model trained using depth up-scaling by Upstage AI",
+)
+
+register_model_info(
+    ["gpt-4-turbo"],
+    "GPT-4-Turbo",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4-vision-preview"],
+    "gpt-4-vision-preview",
+    "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo",
+    "GPT-4(V) by OpenAI",
+)
+
+register_model_info(
+    ["gpt-3.5-turbo", "gpt-3.5-turbo-0314", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-1106"],
+    "GPT-3.5",
+    "https://platform.openai.com/docs/models/gpt-3-5",
+    "GPT-3.5-Turbo by OpenAI",
+)
+
+register_model_info(
+    ["gpt-4", "gpt-4-0314", "gpt-4-0613"],
+    "GPT-4",
+    "https://openai.com/research/gpt-4",
+    "GPT-4 by OpenAI",
+)
+
+register_model_info(
+    ["claude-2.1", "claude-2.0"],
+    "Claude",
+    "https://www.anthropic.com/index/claude-2",
+    "Claude 2 by Anthropic",
+)
+
+register_model_info(
+    ["claude-1"],
+    "Claude",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude 1 by Anthropic",
+)
+
+register_model_info(
+    ["claude-instant-1", "claude-instant-1.2"],
+    "Claude Instant",
+    "https://www.anthropic.com/index/introducing-claude",
+    "Claude Instant by Anthropic",
+)
+
+register_model_info(
+    ["pplx-70b-online", "pplx-7b-online"],
+    "pplx-online-llms",
+    "https://blog.perplexity.ai/blog/introducing-pplx-online-llms",
+    "Online LLM API by Perplexity AI",
+)
+
+register_model_info(
+    ["openhermes-2.5-mistral-7b"],
+    "OpenHermes-2.5-Mistral-7B",
+    "https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B",
+    "a mistral-based model fine-tuned on 1M GPT-4 outputs",
+)
+
+register_model_info(
+    ["starling-lm-7b-alpha"],
+    "Starling-LM-7B-alpha",
+    "https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha",
+    "an open model trained using RLAIF by Berkeley",
+)
+
+register_model_info(
+    ["tulu-2-dpo-70b"],
+    "Tulu 2",
+    "https://huggingface.co/allenai/tulu-2-dpo-70b",
+    "an instruction and RLHF model by UW/AllenAI",
+)
+
+register_model_info(
+    ["yi-34b-chat", "yi-6b-chat"],
+    "Yi-Chat",
+    "https://huggingface.co/01-ai/Yi-34B-Chat",
+    "A large language model by 01 AI",
+)
+
+register_model_info(
+    ["llama-2-70b-chat", "llama-2-34b-chat", "llama-2-13b-chat", "llama-2-7b-chat"],
+    "Llama 2",
+    "https://ai.meta.com/llama/",
+    "open foundation and fine-tuned chat models by Meta",
+)
+
+register_model_info(
+    [
+        "vicuna-33b",
+        "vicuna-33b-v1.3",
+        "vicuna-13b",
+        "vicuna-13b-v1.3",
+        "vicuna-7b",
+        "vicuna-7b-v1.3",
+    ],
+    "Vicuna",
+    "https://lmsys.org/blog/2023-03-30-vicuna/",
+    "a chat assistant fine-tuned on user-shared conversations by LMSYS",
+)
+
+register_model_info(
+    ["chatglm3-6b", "chatglm2-6b", "chatglm-6b"],
+    "ChatGLM",
+    "https://chatglm.cn/blog",
+    "an open bilingual dialogue language model by Tsinghua University",
+)
+
+register_model_info(
+    ["openchat-3.5"],
+    "OpenChat 3.5",
+    "https://github.com/imoneoi/openchat",
+    "an open model fine-tuned on Mistral-7B using C-RLFT",
+)
+
+register_model_info(
+    ["tenyxchat-7b-v1"],
+    "TenyxChat-7B",
+    "https://huggingface.co/tenyx/TenyxChat-7B-v1",
+    "an open model DPO trained on top of OpenChat-3.5 using Tenyx fine-tuning",
+)
+
+register_model_info(
+    ["zephyr-7b-beta", "zephyr-7b-alpha"],
+    "Zephyr",
+    "https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha",
+    "a chatbot fine-tuned from Mistral by Hugging Face",
+)
+
+register_model_info(
+    ["notus-7b-v1"],
+    "Notus",
+    "https://huggingface.co/argilla/notus-7b-v1",
+    "a chatbot fine-tuned from Zephyr SFT by Argilla",
+)
+
+register_model_info(
+    ["catppt"],
+    "CatPPT",
+    "https://huggingface.co/rishiraj/CatPPT",
+    "a chatbot fine-tuned from a SLERP merged model by Rishiraj Acharya",
+)
+
+register_model_info(
+    ["TinyLlama"],
+    "TinyLlama",
+    "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "The TinyLlama project is an open endeavor to pretrain a 1.1B Llama model on 3 trillion tokens.",
+)
+
+register_model_info(
+    ["qwen-14b-chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-14B-Chat",
+    "a large language model by Alibaba Cloud",
+)
+
+register_model_info(
+    ["codellama-34b-instruct", "codellama-13b-instruct", "codellama-7b-instruct"],
+    "Code Llama",
+    "https://ai.meta.com/blog/code-llama-large-language-model-coding/",
+    "open foundation models for code by Meta",
+)
+
+register_model_info(
+    ["wizardlm-70b", "wizardlm-30b", "wizardlm-13b"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM",
+    "an instruction-following LLM using evol-instruct by Microsoft",
+)
+
+register_model_info(
+    ["wizardcoder-15b-v1.0"],
+    "WizardLM",
+    "https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder",
+    "Empowering Code Large Language Models with Evol-Instruct",
+)
+
+register_model_info(
+    ["mpt-7b-chat", "mpt-30b-chat"],
+    "MPT-Chat",
+    "https://www.mosaicml.com/blog/mpt-30b",
+    "a chatbot fine-tuned from MPT by MosaicML",
+)
+
+register_model_info(
+    ["guanaco-33b", "guanaco-65b"],
+    "Guanaco",
+    "https://github.com/artidoro/qlora",
+    "a model fine-tuned with QLoRA by UW",
+)
+
+register_model_info(
+    ["gpt4all-13b-snoozy"],
+    "GPT4All-Snoozy",
+    "https://github.com/nomic-ai/gpt4all",
+    "a finetuned LLaMA model on assistant style data by Nomic AI",
+)
+
+register_model_info(
+    ["koala-13b"],
+    "Koala",
+    "https://bair.berkeley.edu/blog/2023/04/03/koala",
+    "a dialogue model for academic research by BAIR",
+)
+
+register_model_info(
+    ["RWKV-4-Raven-14B"],
+    "RWKV-4-Raven",
+    "https://huggingface.co/BlinkDL/rwkv-4-raven",
+    "an RNN with transformer-level LLM performance",
+)
+
+register_model_info(
+    ["alpaca-13b"],
+    "Alpaca",
+    "https://crfm.stanford.edu/2023/03/13/alpaca.html",
+    "a model fine-tuned from LLaMA on instruction-following demonstrations by Stanford",
+)
+
+register_model_info(
+    ["oasst-pythia-12b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["oasst-sft-7-llama-30b"],
+    "OpenAssistant (oasst)",
+    "https://open-assistant.io",
+    "an Open Assistant for everyone by LAION",
+)
+
+register_model_info(
+    ["palm-2"],
+    "PaLM 2 Chat",
+    "https://cloud.google.com/vertex-ai/docs/release-notes#May_10_2023",
+    "PaLM 2 for Chat (chat-bison@001) by Google",
+)
+
+register_model_info(
+    ["llama-7b", "llama-13b"],
+    "LLaMA",
+    "https://arxiv.org/abs/2302.13971",
+    "open and efficient foundation language models by Meta",
+)
+
+register_model_info(
+    ["open-llama-7b-v2-open-instruct", "open-llama-7b-open-instruct"],
+    "Open LLaMa (Open Instruct)",
+    "https://medium.com/vmware-data-ml-blog/starter-llm-for-the-enterprise-instruction-tuning-openllama-7b-d05fc3bbaccc",
+    "Open LLaMa fine-tuned on instruction-following data by VMware",
+)
+
+register_model_info(
+    ["dolly-v2-12b"],
+    "Dolly",
+    "https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm",
+    "an instruction-tuned open large language model by Databricks",
+)
+
+register_model_info(
+    ["stablelm-tuned-alpha-7b"],
+    "StableLM",
+    "https://github.com/stability-AI/stableLM",
+    "Stability AI language models",
+)
+
+register_model_info(
+    ["codet5p-6b"],
+    "CodeT5p-6b",
+    "https://huggingface.co/Salesforce/codet5p-6b",
+    "Code completion model released by Salesforce",
+)
+
+register_model_info(
+    ["fastchat-t5-3b", "fastchat-t5-3b-v1.0"],
+    "FastChat-T5",
+    "https://huggingface.co/lmsys/fastchat-t5-3b-v1.0",
+    "a chat assistant fine-tuned from FLAN-T5 by LMSYS",
+)
+
+register_model_info(
+    ["phoenix-inst-chat-7b"],
+    "Phoenix-7B",
+    "https://huggingface.co/FreedomIntelligence/phoenix-inst-chat-7b",
+    "a multilingual chat assistant fine-tuned from Bloomz to democratize ChatGPT across languages by CUHK(SZ)",
+)
+
+register_model_info(
+    ["realm-7b-v1"],
+    "ReaLM",
+    "https://github.com/FreedomIntelligence/ReaLM",
+    "A chatbot fine-tuned from LLaMA2 with data generated via iterative calls to UserGPT and ChatGPT by CUHK(SZ) and SRIBD.",
+)
+
+register_model_info(
+    ["billa-7b-sft"],
+    "BiLLa-7B-SFT",
+    "https://huggingface.co/Neutralzz/BiLLa-7B-SFT",
+    "an instruction-tuned bilingual LLaMA with enhanced reasoning ability by an independent researcher",
+)
+
+register_model_info(
+    ["h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2"],
+    "h2oGPT-GM-7b",
+    "https://huggingface.co/h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2",
+    "an instruction-tuned OpenLLaMA with enhanced conversational ability by H2O.ai",
+)
+
+register_model_info(
+    ["baize-v2-7b", "baize-v2-13b"],
+    "Baize v2",
+    "https://github.com/project-baize/baize-chatbot#v2",
+    "A chatbot fine-tuned from LLaMA with ChatGPT self-chat data and Self-Disillation with Feedback (SDF) by UCSD and SYSU.",
+)
+
+register_model_info(
+    [
+        "airoboros-l2-7b-2.1",
+        "airoboros-l2-13b-2.1",
+        "airoboros-c34b-2.1",
+        "airoboros-l2-70b-2.1",
+    ],
+    "airoboros",
+    "https://huggingface.co/jondurbin/airoboros-l2-70b-2.1",
+    "an instruction-tuned LlaMa model tuned with 100% synthetic instruction-response pairs from GPT4",
+)
+
+register_model_info(
+    [
+        "spicyboros-7b-2.2",
+        "spicyboros-13b-2.2",
+        "spicyboros-70b-2.2",
+    ],
+    "spicyboros",
+    "https://huggingface.co/jondurbin/spicyboros-70b-2.2",
+    "de-aligned versions of the airoboros models",
+)
+
+register_model_info(
+    ["Robin-7b-v2", "Robin-13b-v2", "Robin-33b-v2"],
+    "Robin-v2",
+    "https://huggingface.co/OptimalScale/robin-7b-v2-delta",
+    "A chatbot fine-tuned from LLaMA-7b, achieving competitive performance on chitchat, commonsense reasoning and instruction-following tasks, by OptimalScale, HKUST.",
+)
+
+register_model_info(
+    ["manticore-13b-chat"],
+    "Manticore 13B Chat",
+    "https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg",
+    "A chatbot fine-tuned from LlaMa across several CoT and chat datasets.",
+)
+
+register_model_info(
+    ["redpajama-incite-7b-chat"],
+    "RedPajama-INCITE-7B-Chat",
+    "https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat",
+    "A chatbot fine-tuned from RedPajama-INCITE-7B-Base by Together",
+)
+
+register_model_info(
+    [
+        "falcon-7b",
+        "falcon-7b-instruct",
+        "falcon-40b",
+        "falcon-40b-instruct",
+        "falcon-180b",
+        "falcon-180b-chat",
+    ],
+    "Falcon",
+    "https://huggingface.co/tiiuae/falcon-180B",
+    "TII's flagship series of large language models",
+)
+
+register_model_info(
+    ["tigerbot-7b-sft"],
+    "Tigerbot",
+    "https://huggingface.co/TigerResearch/tigerbot-7b-sft",
+    "TigerBot is a large-scale language model (LLM) with multiple languages and tasks.",
+)
+
+register_model_info(
+    ["internlm-chat-7b", "internlm-chat-7b-8k"],
+    "InternLM",
+    "https://huggingface.co/internlm/internlm-chat-7b",
+    "InternLM is a multi-language large-scale language model (LLM), developed by SHLAB.",
+)
+
+register_model_info(
+    ["Qwen-7B-Chat"],
+    "Qwen",
+    "https://huggingface.co/Qwen/Qwen-7B-Chat",
+    "Qwen is a multi-language large-scale language model (LLM), developed by Damo Academy.",
+)
+
+register_model_info(
+    ["Llama2-Chinese-13b-Chat", "LLama2-Chinese-13B"],
+    "Llama2-Chinese",
+    "https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat",
+    "Llama2-Chinese is a multi-language large-scale language model (LLM), developed by FlagAlpha.",
+)
+
+register_model_info(
+    ["Chinese-Alpaca-2-7B", "Chinese-Alpaca-2-13B"],
+    "Chinese-Alpaca",
+    "https://huggingface.co/hfl/chinese-alpaca-2-13b",
+    "New extended Chinese vocabulary beyond Llama-2, open-sourcing the Chinese LLaMA-2 and Alpaca-2 LLMs.",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Instruct", "Vigogne-2-13B-Instruct"],
+    "Vigogne-Instruct",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-instruct",
+    "Vigogne-Instruct is a French large language model (LLM) optimized for instruction-following, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["Vigogne-2-7B-Chat", "Vigogne-2-13B-Chat"],
+    "Vigogne-Chat",
+    "https://huggingface.co/bofenghuang/vigogne-2-7b-chat",
+    "Vigogne-Chat is a French large language model (LLM) optimized for instruction-following and multi-turn dialogues, developed by Bofeng Huang",
+)
+
+register_model_info(
+    ["stable-vicuna-13B-HF"],
+    "stable-vicuna",
+    "https://huggingface.co/TheBloke/stable-vicuna-13B-HF",
+    "StableVicuna is a Vicuna model fine-tuned using RLHF via PPO on various conversational and instructional datasets.",
+)
+
+register_model_info(
+    ["deluxe-chat-v1", "deluxe-chat-v1.1", "deluxe-chat-v1.2"],
+    "DeluxeChat",
+    "",
+    "Deluxe Chat",
+)
+
+register_model_info(
+    [
+        "Xwin-LM-7B-V0.1",
+        "Xwin-LM-13B-V0.1",
+        "Xwin-LM-70B-V0.1",
+        "Xwin-LM-7B-V0.2",
+        "Xwin-LM-13B-V0.2",
+    ],
+    "Xwin-LM",
+    "https://github.com/Xwin-LM/Xwin-LM",
+    "Chat models developed by Xwin-LM team",
+)
+
+register_model_info(
+    ["lemur-70b-chat"],
+    "Lemur-Chat",
+    "https://huggingface.co/OpenLemur/lemur-70b-chat-v1",
+    "an openly accessible language model optimized for both natural language and coding capabilities ",
+)
+
+register_model_info(
+    ["Mistral-7B-OpenOrca"],
+    "Open-Orca",
+    "https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca",
+    "A fine-tune of [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) using [OpenOrca dataset](https://huggingface.co/datasets/Open-Orca/OpenOrca)",
+)
+
+register_model_info(
+    ["dolphin-2.2.1-mistral-7b"],
+    "dolphin-mistral",
+    "https://huggingface.co/ehartford/dolphin-2.2.1-mistral-7b",
+    "An uncensored fine-tuned Mistral 7B",
+)
+
+register_model_info(
+    [
+        "AquilaChat-7B",
+        "AquilaChat2-7B",
+        "AquilaChat2-34B",
+    ],
+    "Aquila-Chat",
+    "https://huggingface.co/BAAI/AquilaChat2-34B",
+    "Chat models developed by BAAI team",
+)
+
+register_model_info(
+    ["xDAN-L1-Chat-RL-v1"],
+    "xDAN-L1-Chat",
+    "https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1",
+    "A large language chat model created by xDAN-AI.",
+)
+
+register_model_info(
+    ["MetaMath-70B-V1.0", "MetaMath-7B-V1.0"],
+    "MetaMath",
+    "https://huggingface.co/meta-math",
+    "MetaMath is a finetune of Llama2 on [MetaMathQA](https://huggingface.co/datasets/meta-math/MetaMathQA) that specializes in mathematical reasoning.",
+)
+
+register_model_info(
+    ["Yuan2-2B-hf", "Yuan2-51B-hf", "Yuan2-102B-hf"],
+    "IEIYuan",
+    "https://huggingface.co/IEITYuan",
+    "Yuan2 is a Basemodel developed by IEI.",
+)

arena_elo/elo_rating/upload_battle_data.py

@@ -0,0 +1,193 @@
+import fire
+import json
+import os
+import datasets
+import datetime
+from pathlib import Path
+from datetime import datetime
+from PIL import Image
+
+datasets.config.DEFAULT_MAX_BATCH_SIZE = 500
+def create_hf_dataset(data_file: str, split="test"):
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=datasets.Features(
+            {
+                "question_id": datasets.Value("string"),
+                "model": datasets.Value("string"),
+                "conversation": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
+            }
+        ),
+        split=split,
+    )
+    return hf_dataset
+
+def create_hf_battle_dataset(data_file: str, split="test"):
+    hf_dataset = datasets.Dataset.from_list(
+        data_file,
+        features=datasets.Features(
+            {
+                "question_id": datasets.Value("string"),
+                "model_a": datasets.Value("string"),
+                "model_b": datasets.Value("string"),
+                "conversation_a": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "conversation_b": [
+                    {
+                        "role": datasets.Value("string"),
+                        "content": datasets.Value("string"),
+                    }
+                ],
+                "language": datasets.Value("string"),
+                "image": datasets.Image(),
+                "turn": datasets.Value("int32"),
+                "anony": datasets.Value("bool"),
+            }
+        ),
+        split=split,
+    )
+    return hf_dataset
+                           
+                                            
+
+
+def load_image(path:str):
+    try:
+        return Image.open(path)
+    except Exception as e:
+        print(f"Error loading image {path}: {e}")
+        return None
+
+def get_date_from_time_stamp(unix_timestamp: int):
+    # Create a datetime object from the Unix timestamp
+    dt = datetime.fromtimestamp(unix_timestamp)
+
+    # Convert the datetime object to a string with the desired format
+    date_str = dt.strftime("%Y-%m-%d")
+    return date_str
+
+def load_battle_image(battle, log_dir):
+    image_path = Path(log_dir) / f"{get_date_from_time_stamp(battle['tstamp'])}-convinput_images" / f"input_image_{battle['question_id']}.png"
+    return load_image(image_path)
+    
+
+def main(
+    data_file: str = "./results/latest/clean_battle_conv.json",
+    repo_id: str = "DongfuTingle/wildvision-bench",
+    log_dir: str = os.getenv("LOGDIR", "./vision-arena-logs/"),
+    mode="battle",
+    token = os.environ.get("HUGGINGFACE_TOKEN", None)
+):
+    with open(data_file, "r") as f:
+        data = json.load(f)
+    
+    
+    
+    has_image_stats = {
+        "has_image": 0,
+        "no_image": 0,
+    }
+    if mode == "keep_bad_only":
+        # anony only
+        data = [d for d in data if d["anony"]]
+        
+        new_data = []
+        for battle in data:
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
+                # we don't keep the data without image
+                continue
+            has_image_stats["has_image"] += 1
+            
+            if battle["winner"] in ["model_a", "model_b"]:
+                if battle["winner"] == "model_a":
+                    worse_model = "model_b"
+                    worse_conv = "conversation_b"
+                if battle["winner"] == "model_b":
+                    worse_model = "model_a"
+                    worse_conv = "conversation_a"
+                    
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle[worse_model],
+                    "conversation": battle[worse_conv],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+            elif battle["winner"] == "tie (bothbad)":
+                
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_a"],
+                    "conversation": battle["conversation_a"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+
+                new_data.append({
+                    "question_id": battle["question_id"],
+                    "model": battle["model_b"],
+                    "conversation": battle["conversation_b"],
+                    "language": battle["language"],
+                    "image": image,
+                    "turn": battle["turn"],
+                })
+                
+        split = "test"
+        hf_dataset = create_hf_dataset(new_data, "test")
+    
+    elif mode == "battle":
+        new_data = []
+        for battle in data:
+            image = load_battle_image(battle, log_dir)
+            if image is None:
+                has_image_stats["no_image"] += 1
+                continue
+            has_image_stats["has_image"] += 1
+            new_data.append({
+                "question_id": battle["question_id"],
+                "model_a": battle["model_a"],
+                "model_b": battle["model_b"],
+                "conversation_a": battle["conversation_a"],
+                "conversation_b": battle["conversation_b"],
+                "language": battle["language"],
+                "image": image,
+                "turn": battle["turn"],
+                "anony": battle["anony"],
+            })
+        split = "test"
+        hf_dataset = create_hf_battle_dataset(new_data, "test")
+    else:
+        raise ValueError(f"Invalid mode: {mode}")
+
+    print(f"Stats: {has_image_stats}")
+    print(hf_dataset)
+    print(f"Uploading to part {repo_id}:{split}...")
+    hf_dataset.push_to_hub(
+        repo_id=repo_id,
+        config_name=mode,
+        split=split,
+        token=token,
+        commit_message=f"Add vision-arena {split} dataset",
+    )
+    
+    print("Done!")
+    
+    
+if __name__ == "__main__":
+    fire.Fire(main)

arena_elo/elo_rating/utils.py

@@ -0,0 +1,83 @@
+from datetime import datetime
+import pytz
+import PIL
+import os 
+
+def detect_language(text: str) -> str:
+    """Detect the langauge of a string."""
+    import polyglot  # pip3 install polyglot pyicu pycld2
+    from polyglot.detect import Detector
+    from polyglot.detect.base import logger as polyglot_logger
+    import pycld2
+
+    polyglot_logger.setLevel("ERROR")
+
+    try:
+        lang_code = Detector(text).language.name
+    except (pycld2.error, polyglot.detect.base.UnknownLanguage):
+        lang_code = "unknown"
+    return lang_code
+
+
+def get_time_stamp_from_date(date_str:str):
+    """
+    Convert a date string to a Unix timestamp
+    Args:
+        date_str (str): The input date string in the format 'YYYY-MM-DD-HH:MM-TZ', e.g. '2024-02-10-14:00-PT'
+    """
+    
+    # Convert the date string into a format that Python's datetime can understand
+    # and specify the correct timezone for PT, which is 'US/Pacific'
+    date_format = "%Y-%m-%d-%H:%M-%Z"
+
+    # Parse the date string into a datetime object
+    # Note: PT is not directly recognized by pytz, so we manually map it to 'US/Pacific'
+    timezone_map = {
+        "PT": "US/Pacific",
+    }
+
+    # Extract the timezone abbreviation
+    tz_abbr = date_str.split("-")[-1]
+    # Map the abbreviation to a pytz timezone
+    tz_info = pytz.timezone(timezone_map[tz_abbr])
+
+    # Remove the timezone abbreviation for parsing
+    date_str_parsed = date_str.rsplit("-", 1)[0]
+
+    # Create a datetime object with the corresponding timezone
+    dt = datetime.strptime(date_str_parsed, "%Y-%m-%d-%H:%M").replace(tzinfo=tz_info)
+
+    # Convert the datetime object to a Unix timestamp
+    unix_timestamp = dt.timestamp()
+    return unix_timestamp
+
+def get_date_from_time_stamp(unix_timestamp: int):
+    # Create a datetime object from the Unix timestamp
+    dt = datetime.fromtimestamp(unix_timestamp)
+
+    # Convert the datetime object to a string with the desired format
+    date_str = dt.strftime("%Y-%m-%d %H:%M:%S %Z")
+    return date_str
+
+
+def get_input_image_path(tstamp, conv_id):
+    # from tstamp to date e.g. 2024-02-10
+    date_str = datetime.fromtimestamp(tstamp, tz=pytz.timezone("US/Pacific")).strftime("%Y-%m-%d")
+    LOGDIR = os.getenv("LOGDIR")
+    return f"{LOGDIR}/{date_str}-convinput_images/input_image_{conv_id}.png"
+
+def load_image_from_path(image_path):
+    # Load the image from the specified
+    # path using the Python Imaging Library (PIL)
+    try:
+        image = PIL.Image.open(image_path)
+        return image
+    except FileNotFoundError:
+        print(f"Image not found at path: {image_path}")
+        return None
+    except PIL.UnidentifiedImageError:
+        print(f"Unidentified image format at path: {image_path}")
+        return None
+    
+
+    

arena_elo/evaluator/convert_to_evaluator_data.py

@@ -0,0 +1,134 @@
+import argparse
+import json
+import os
+import time
+from pytz import timezone
+from tqdm import tqdm
+import base64
+from icecream import ic
+from PIL import Image
+
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+def get_log_files(max_num_files=None):
+    dates = []
+    for month in [2, 3]:
+        for day in range(1, 32):
+            dates.append(f"2024-{month:02d}-{day:02d}")
+
+    num_servers = 1
+    filenames = []
+    for d in dates:
+        for i in range(num_servers):
+            # name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
+            name = os.path.expanduser(f"vision-arena-logs/{d}-conv.json")
+            if os.path.exists(name):
+                filenames.append(name)
+    max_num_files = max_num_files or len(filenames)
+    filenames = filenames[-max_num_files:]
+    return filenames
+
+
+def pretty_print_conversation(messages):
+    for role, msg in messages:
+        print(f"[[{role}]]: {msg}")
+
+task_template_map = {
+    "image_caption": "Give me the semantic alignment score between the given image and the given caption: \"{generated_sentence}\" on a scale of 0-100. Only reply the score value.",
+    "vqa": "Rate the answer correctness regarding the question within the context of the given image on a scale of 0-100. Only reply the score value.",
+    "pair_rate_old": "[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nGiven the instruction and the image, please compare the correctness of responses A and B. Reply with \"leftvote\" if you find A better, \"rightvote\" if B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both responses are equally satisfactory. If you are unable to make a decision, please reply with \"NA\".",
+    "pair_rate_wexplanation": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.",
+    "pair_rate": "<image>[Instruction]\n\"{instruction}\"\n\n\"{generated_sentence}\"\n\n[System]\nPlease act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user’s instructions and answers the user’s question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any positional biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. Reply with \"leftvote\" if you find assistant A better, \"rightvote\" if assistant B is better, \"bothbad_vote\" if both responses are wrong, and \"tievote\" if both assistants provide equally satisfactory answers. If you are unable to make a decision, please reply with \"NA\"."
+}
+
+def inspect_convs(log_files):
+    json_data = []
+
+    ic(log_files)
+    total_vote = 0
+
+    for filename in tqdm(log_files, desc="read files"):
+        for retry in range(5):
+            try:
+                lines = open(filename).readlines()
+                break
+            except FileNotFoundError:
+                time.sleep(2)
+        
+        for l in lines:
+            row = json.loads(l)
+            
+            if "states" not in row:
+                continue
+            if row["type"] not in ["leftvote", "rightvote", "bothbad_vote", "tievote"]:
+                continue
+            
+            model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
+            
+
+            # Iterate through each state and write the relevant information
+            if not len(row["states"][0]['messages']): continue
+            # ic(row["states"][0]['messages'][1][1])
+
+            if row["states"][0]['messages'][1][1] is None or row["states"][1]['messages'][1][1] is None or "NETWORK ERROR" in row["states"][0]['messages'][1][1] or "NETWORK ERROR" in row["states"][1]['messages'][1][1]: continue
+            total_vote += 1
+
+            conv_id = row["states"][0]['conv_id']
+            image_path = os.path.join("/local/home/yujielu/project/Arena-Elo/vision-arena-logs", os.path.basename(filename)[:-5]+"input_images", f"input_image_{conv_id}.png")
+            if not os.path.exists(image_path) :
+                continue
+            try:
+                image = Image.open(image_path).convert("RGB")
+            except:
+                continue
+
+            left_response = row["states"][0]['messages'][1][1]
+            right_response = row["states"][1]['messages'][1][1]
+            instruction = row["states"][0]['messages'][0][1]
+            generated_sentence = f"[The Start of Assistant A’s Answer]\n{left_response}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{right_response}\n[The End of Assistant B’s Answer]"
+            text_prompt = task_template_map["pair_rate"].format(instruction=instruction, generated_sentence=generated_sentence)
+            
+            user_input = text_prompt
+            # Create the conversation structure
+            conversation = [
+                {
+                    "from": "human",
+                    "value": user_input
+                },
+                {
+                    "from": "gpt",
+                    "value": row["type"]
+                }
+            ]
+            
+            # Create the JSON object for each row
+            json_obj = {
+                "id": conv_id,
+                "image": image_path,
+                "conversations": conversation
+            }
+            
+            # Append the JSON object to the list
+            json_data.append(json_obj)
+
+        # Write the JSON data to a file
+        with open('output_evaluator_data.json', 'w') as json_file:
+            json.dump(json_data, json_file, indent=2)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--max-num-files", type=int)
+    args = parser.parse_args()
+
+    log_files = get_log_files(args.max_num_files)
+    
+
+            
+    inspect_convs(log_files)
+
+
+

constants.py

@@ -26,4 +26,4 @@ TEXT_PROMPT_PATH = "offline/prompts_110.json"
 IMAGE_PROMPT_PATH = "offline/image_urls.txt"
 
 MAX_ATTEMPTS = 5
-REPLICATE_API_TOKEN = os.getenv("REPLICATE_API_TOKEN", "r8_0BaoQW0G8nWFXY8YWBCCUDurANxCtY72rarv9")
+REPLICATE_API_TOKEN = os.getenv("REPLICATE_API_TOKEN", "yourKey")

model/model_worker.py

@@ -6,8 +6,10 @@ from typing import List
 import replicate
 import subprocess
 
-from constants import OFFLINE_GIF_DIR
-# os.environ("REPLICATE_API_TOKEN", "r8_0BaoQW0G8nWFXY8YWBCCUDurANxCtY72rarv9")
+from gradio_client import Client
+from client import Gau2Mesh_client
+from constants import OFFLINE_GIF_DIR, REPLICATE_API_TOKEN
+# os.environ("REPLICATE_API_TOKEN", "yourKey")
 
 class BaseModelWorker:
     def __init__(self,

serve/inference.py

@@ -188,14 +188,17 @@ def generate_t2s(gen_func, render_func,
         state.rgb_video = videos['rgb']        
         yield state, videos['normal'], videos['rgb']
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data = {
-            "ip": ip,
+            "tstamp": round(finish_tstamp, 4),
+            "ip": get_ip(request),
             "model": model_name,
             "type": "offline",
             "gen_params": {},
             "state": state.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
     else:
         start_time = time.time()
@@ -210,14 +213,17 @@ def generate_t2s(gen_func, render_func,
         state.rgb_video = videos['rgb']    
         yield state, videos['normal'], videos['rgb']
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data = {
-            "ip": ip,
+            "tstamp": round(finish_tstamp, 4),
+            "ip": get_ip(request),
             "model": model_name,
             "type": "online",
             "gen_params": {},
             "state": state.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
@@ -277,22 +283,27 @@ def generate_t2s_multi(gen_func, render_func,
         state_1.normal_video, state_1.rgb_video = videos_1['normal'], videos_1['rgb']
         yield state_0, state_1,videos_0['normal'], videos_0['rgb'], videos_1['normal'], videos_1['rgb']
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "offline",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "offline",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
     else:
         start_time = time.time()
@@ -307,25 +318,30 @@ def generate_t2s_multi(gen_func, render_func,
         state_1.normal_video, state_1.rgb_video = videos_1['normal'], videos_1['rgb']
         yield state_0, state_1, videos_0['normal'], videos_0['rgb'], videos_1['normal'], videos_1['rgb']
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "online",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "online",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
@@ -386,22 +402,27 @@ def generate_t2s_multi_annoy(gen_func, render_func,
         yield state_0, state_1, videos_0['normal'], videos_0['rgb'], videos_1['normal'], videos_1['rgb'], \
             gr.Markdown(f"### Model A: {model_name_0}"), gr.Markdown(f"### Model B: {model_name_1}")
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "offline",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "offline",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
     else:
         start_time = time.time()
@@ -418,25 +439,30 @@ def generate_t2s_multi_annoy(gen_func, render_func,
         yield state_0, state_1, videos_0[0], videos_0[1], videos_1[0], videos_1[1], \
             gr.Markdown(f"### Model A: {model_name_0}"), gr.Markdown(f"### Model B: {model_name_1}")
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "online",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "online",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
@@ -481,14 +507,17 @@ def generate_i2s(gen_func, render_func, state, image, model_name, request: gr.Re
         state.rgb_video = videos['rgb']
         yield state, videos['normal'], videos['rgb']
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data = {
-            "ip": ip,
+            "tstamp": round(finish_tstamp, 4),
+            "ip": get_ip(request),
             "model": model_name,
             "type": "offline",
             "gen_params": {},
             "state": state.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
     else:
         start_time = time.time()
@@ -503,14 +532,17 @@ def generate_i2s(gen_func, render_func, state, image, model_name, request: gr.Re
         state.rgb_video = videos['rgb']
         yield state, videos['normal'], videos['rgb']
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data = {
-            "ip": ip,
+            "tstamp": round(finish_tstamp, 4),
+            "ip": get_ip(request),
             "model": model_name,
             "type": "online",
             "gen_params": {},
             "state": state.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
@@ -567,22 +599,27 @@ def generate_i2s_multi(gen_func, render_func,
         yield state_0, state_1, videos_0['normal'], videos_0['rgb'], videos_1['normal'], videos_1['rgb'], \
             gr.Markdown(f"### Model A: {model_name_0}"), gr.Markdown(f"### Model B: {model_name_1}")
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "offline",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "offline",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
     else:
         start_time = time.time()
@@ -597,25 +634,30 @@ def generate_i2s_multi(gen_func, render_func,
         state_1.normal_video, state_1.rgb_video = videos_1['normal'], videos_1['rgb']
         yield state_0, state_1, videos_0['normal'], videos_0['rgb'], videos_1['normal'], videos_1['rgb']
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "online",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "online",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
@@ -672,26 +714,31 @@ def generate_i2s_multi_annoy(gen_func, render_func,
         yield state_0, state_1, videos_0['normal'], videos_0['rgb'], videos_1['normal'], videos_1['rgb'], \
             gr.Markdown(f"### Model A: {model_name_0}"), gr.Markdown(f"### Model B: {model_name_1}")
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "offline",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "offline",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
         }
     else:
         start_time = time.time()
-        shape_0, shape_1 = gen_func(image, model_name_0, model_name_1)
+        shape_0, shape_1 = gen_func(image, model_name_0, model_name_1, i2s_model=True)
         generate_time = time.time() - start_time
 
         videos_0, videos_1 = render_func(shape_0, model_name_0, shape_1, model_name_1)
@@ -703,25 +750,30 @@ def generate_i2s_multi_annoy(gen_func, render_func,
         yield state_0, state_1,  videos_0['normal'], videos_0['rgb'], videos_1['normal'], videos_1['rgb'], \
             gr.Markdown(f"### Model A: {model_name_0}"), gr.Markdown(f"### Model B: {model_name_1}")
 
+        finish_tstamp = time.time()
         # logger.info(f"===output===: {output}")
         data_0 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_0,
             "type": "online",
             "gen_params": {},
             "state": state_0.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),
         }
         data_1 = {
+            "tstamp": round(finish_tstamp, 4),
             "ip": get_ip(request),
             "model": model_name_1,
             "type": "online",
             "gen_params": {},
             "state": state_1.dict(),
             "start": round(start_time, 4),
+            "finish": round(finish_tstamp, 4),
             "time": round(finish_time - start_time, 4),
             "generate_time": round(generate_time, 4),
             "render_time": round(render_time, 4),

serve/leaderboard.py

@@ -39,8 +39,6 @@ leader_component_values = [None] * 5
 def make_leaderboard_md(elo_results):
     leaderboard_md = f"""
 # 🏆 GenAI-Arena Leaderboard
-| [GitHub](https://github.com/TIGER-AI-Lab/ImagenHub) | [Dataset](https://huggingface.co/ImagenHub) | [Twitter](https://twitter.com/TianleLI123/status/1757245259149422752) |
-
 """
     return leaderboard_md
 

serve/vote_utils.py

@@ -26,12 +26,13 @@ def vote_last_response_t2s(state, dim, vote_type, model_selector, request: gr.Re
         fout.write(json.dumps(data) + "\n")
         append_json_item_on_log_server(data, get_conv_log_filename())
         
-def vote_last_response_t2s_multi(states, dim, vote_type, model_selectors, request: gr.Request):
+def vote_last_response_t2s_multi(states, dim, vote_type, is_anony: bool, model_selectors, request: gr.Request):
     with open(get_conv_log_filename(), "a") as fout:
         data = {
             "tstamp": round(time.time(), 4),
             "dim": dim,
             "type": vote_type,
+            "anony": is_anony,
             "models": [x for x in model_selectors],
             "states": [x.dict() for x in states],
             "ip": get_ip(request),
@@ -65,12 +66,13 @@ def vote_last_response_i2s(state, dim, vote_type, model_selector, request: gr.Re
     # save_image_file_on_log_server(output_file)
     # save_image_file_on_log_server(source_file)
         
-def vote_last_response_i2s_multi(states, dim, vote_type, model_selectors, request: gr.Request):
+def vote_last_response_i2s_multi(states, dim, vote_type, is_anony, model_selectors, request: gr.Request):
     with open(get_conv_log_filename(), "a") as fout:
         data = {
             "tstamp": round(time.time(), 4),
             "dim": dim,
             "type": vote_type,
+            "anony": is_anony,
             "models": [x for x in model_selectors],
             "states": [x.dict() for x in states],
             "ip": get_ip(request),
@@ -91,23 +93,23 @@ def vote_last_response_i2s_multi(states, dim, vote_type, model_selectors, reques
 ## Text-to-Shape Generation (t2s) Single Model Direct Chat
 def upvote_last_response_t2s(state, model_selector, dim_md, request: gr.Request):
     ip = get_ip(request)
+    state.evaluted_dims += 1
     t2s_logger.info(f"upvote [{dim_md}]. ip: {ip}")
     vote_last_response_t2s(state, dim_md, "upvote", model_selector, request)
-    state.evaluted_dims += 1
     return (state,) + (disable_btn,) * 3
 
 def downvote_last_response_t2s(state, model_selector, dim_md, request: gr.Request):
     ip = get_ip(request)
+    state.evaluted_dims += 1
     t2s_logger.info(f"downvote [{dim_md}]. ip: {ip}")
     vote_last_response_t2s(state, dim_md, "downvote", model_selector, request)
-    state.evaluted_dims += 1
     return (state,) + (disable_btn,) * 3
 
 def flag_last_response_t2s(state, model_selector, dim_md, request: gr.Request):
     ip = get_ip(request)
+    state.evaluted_dims += 1
     t2s_logger.info(f"flag [{dim_md}]. ip: {ip}")
     vote_last_response_t2s(state, dim_md, "flag", model_selector, request)
-    state.evaluted_dims += 1
     return (state,) + (disable_btn,) * 3
 
 
@@ -115,132 +117,153 @@ def flag_last_response_t2s(state, model_selector, dim_md, request: gr.Request):
 def leftvote_last_response_t2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"leftvote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "leftvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "leftvote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
 
 def rightvote_last_response_t2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"rightvote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "rightvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "rightvote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
 
 def tievote_last_response_t2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"tievote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "tievote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "tievote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
 
 def bothbad_vote_last_response_t2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"bothbad_vote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "bothbad_vote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "bothbad_vote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
 
 
 def leftvote_last_response_t2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"leftvote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "leftvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "leftvote", True, [model_selector0, model_selector1], request
     )
 
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=False), gr.Markdown(f"### Model B: {state1.model_name}", visible=False))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
 
 def rightvote_last_response_t2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"rightvote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "rightvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "rightvote", True, [model_selector0, model_selector1], request
     )
 
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
 
 def tievote_last_response_t2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"tievote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "tievote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "tievote", True, [model_selector0, model_selector1], request
     )
 
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
     
 def bothbad_vote_last_response_t2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     t2s_multi_logger.info(f"bothbad_vote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_t2s_multi(
-        [state0, state1], dim_md, "bothbad_vote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "bothbad_vote", True, [model_selector0, model_selector1], request
     )
     
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
 
 ## Image-to-Shape (i2s) Single Model Direct Chat
 def upvote_last_response_i2s(state, model_selector, dim_md, request: gr.Request):
     ip = get_ip(request)
+    state.evaluted_dims += 1
     i2s_logger.info(f"upvote [{dim_md}]. ip: {ip}")
     vote_last_response_i2s(state, dim_md, "upvote", model_selector, request)
-    state.evaluted_dims += 1
     return (state,) + (disable_btn,) * 3
 
 def downvote_last_response_i2s(state, model_selector, dim_md, request: gr.Request):
     ip = get_ip(request)
+    state.evaluted_dims += 1
     i2s_logger.info(f"downvote [{dim_md}]. ip: {ip}")
     vote_last_response_i2s(state, dim_md, "downvote", model_selector, request)
-    state.evaluted_dims += 1
     return (state,) + (disable_btn,) * 3
 
 def flag_last_response_i2s(state, model_selector, dim_md, request: gr.Request):
     ip = get_ip(request)
+    state.evaluted_dims += 1
     i2s_logger.info(f"flag [{dim_md}]. ip: {ip}")
     vote_last_response_i2s(state, dim_md, "flag", model_selector, request)
-    state.evaluted_dims += 1
     return (state,) + (disable_btn,) * 3
 
 
@@ -248,114 +271,134 @@ def flag_last_response_i2s(state, model_selector, dim_md, request: gr.Request):
 def leftvote_last_response_i2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"leftvote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "leftvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "leftvote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
         
 def rightvote_last_response_i2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"rightvote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "rightvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "rightvote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
 
 def tievote_last_response_i2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"tievote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "tievote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "tievote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
 
 def bothbad_vote_last_response_i2s_named(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"bothbad_vote [{dim_md}] (named). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "bothbad_vote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "bothbad_vote", False, [model_selector0, model_selector1], request
     )
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
     return (state0, state1) + (disable_btn,) * 4
 
 
 def leftvote_last_response_i2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"leftvote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "leftvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "leftvote", True, [model_selector0, model_selector1], request
     )
 
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
         
 
 def rightvote_last_response_i2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"rightvote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "rightvote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "rightvote", True, [model_selector0, model_selector1], request
     )
 
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
         
 
 def tievote_last_response_i2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"tievote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "tievote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "tievote", True, [model_selector0, model_selector1], request
     )
     
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
         
 
 def bothbad_vote_last_response_i2s_anony(
     state0, state1, model_selector0, model_selector1, dim_md, request: gr.Request
 ):
+    state0.evaluted_dims += 1
+    state1.evaluted_dims += 1
     i2s_multi_logger.info(f"bothbad_vote [{dim_md}] (anony). ip: {get_ip(request)}")
     vote_last_response_i2s_multi(
-        [state0, state1], dim_md, "bothbad_vote", [model_selector0, model_selector1], request
+        [state0, state1], dim_md, "bothbad_vote", True, [model_selector0, model_selector1], request
     )
     
-    state0.evaluted_dims += 1
-    state1.evaluted_dims += 1
-    if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
-        names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-        return (state0, state1) + (disable_btn,) * 4 + names
-    else:
-        return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    # if state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS:
+    #     names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    #     return (state0, state1) + (disable_btn,) * 4 + names
+    # else:
+    #     return (state0, state1) + (disable_btn,) * 4 + ("", "")
+    is_visible = (state0.evaluted_dims == state1.evaluted_dims == EVALUATE_DIMS)
+    return (state0, state1) \
+        + (disable_btn,) * 4 \
+        + (gr.Markdown(f"### Model A: {state0.model_name}", visible=is_visible), 
+           gr.Markdown(f"### Model B: {state1.model_name}", visible=is_visible))
         
 
 
@@ -383,13 +426,13 @@ def share_click_t2s_multi(state0, state1, model_selector0, model_selector1, requ
     t2s_multi_logger.info(f"share (anony). ip: {get_ip(request)}")
     if state0 is not None and state1 is not None:
         vote_last_response_t2s_multi(
-            [state0, state1], "share", [model_selector0, model_selector1], request
+            [state0, state1], "", "share", True, [model_selector0, model_selector1], request
         )
 
 def share_click_i2s_multi(state0, state1, model_selector0, model_selector1, request: gr.Request):
     i2s_multi_logger.info(f"share (anony). ip: {get_ip(request)}")
     if state0 is not None and state1 is not None:
         vote_last_response_i2s_multi(
-            [state0, state1], "share", [model_selector0, model_selector1], request
+            [state0, state1], "", "share", True, [model_selector0, model_selector1], request
         )