Add column to data

app.py

@@ -196,20 +196,16 @@ with gr.Blocks(title='3D Animation Arena', head=head, css_paths='static/style.cs
         - GVHMR (https://github.com/zju3dv/GVHMR)
         - HybrIK (https://github.com/jeffffffli/HybrIK)
         - WHAM (https://github.com/yohanshin/WHAM)
-        - CameraHMR (https://github.com/pixelite1201/CameraHMR)
-        - STAF (https://github.com/yw0208/STAF)
-        - TokenHMR (https://github.com/saidwivedi/TokenHMR)
                 
         All inferences are precomputed following the code in the associated GitHub repository. 
         Some post-inference modifications have been made to some models in order to make the comparison possible. 
         These modifications include:
         * Adjusting height to a common ground
         * Fixing the root depth of certain models, when depth was extremely jittery
+        * Fixing the root position of certain models, when no root position was available
 
         All models use the SMPL body model to discard the influence of the body model on the comparison.
         These choices were made without any intention to favor or harm any model. 
-
-        The videos were selected to tests models on a large variety of motions, don't hesitate to send me your videos if you want to have it uploaded in the arena! 
         All matchups are generated randomly, don't hesitate to rate the same videos multiple times as the matchups will probably be different!
 
         ---
@@ -307,6 +303,7 @@ with gr.Blocks(title='3D Animation Arena', head=head, css_paths='static/style.cs
     async def process_rating(state, i, criteria):
         return gr.update(value=await submit_rating(
             criteria=criteria, 
+            video=state['video'],
             winner=state['modelLeft'] if i == 0 else state['modelRight'] if i == 2 else None, 
             loser=state['modelRight'] if i == 0 else state['modelLeft'] if i == 2 else None,
             uuid=state['uuid']

utils.py

@@ -0,0 +1,178 @@
+from typing import Tuple
+import pandas as pd
+import numpy as np
+import time
+import asyncio
+from utils.s3_utils import write_to_s3
+from utils.data_utils import generate_leaderboard, generate_data
+
+submit_lock = asyncio.Lock()
+
+def update_ratings(R_win : int, R_lose : int, k : int = 32) -> Tuple[int, int]:
+    """
+    Update the ratings of two players after a match.
+
+    Args:
+        R_win (int): The rating of the winning player.
+        R_lose (int): The rating of the losing player.
+        k (int, optional): The k-factor. Defaults to 32.
+
+    Returns:
+        Tuple[int, int]: The updated ratings of the winning and losing players.
+    """
+    E_win = 1 / (1 + 10 ** ((R_lose - R_win) / 480))
+    E_lose = 1 / (1 + 10 ** ((R_win - R_lose) / 480))
+    return int(R_win + k * (1 - E_win)), int(R_lose + k * (0 - E_lose))
+
+def generate_matchup(leaderboard : pd.DataFrame, beta : int) -> tuple[str, str]:
+    """
+    Generate a pseudo-random matchup between two models.
+
+    Args:
+        leaderboard (pd.DataFrame): The leaderboard of models
+        beta (int): The damping factor for the Elo update.
+
+    Returns:
+        model1 (str): The first model.
+        model2 (str): The second model.
+    """
+    if leaderboard['Matches'].sum() == 0:
+        return np.random.choice(leaderboard.index, 2, replace=False)
+    weights = [np.exp(-leaderboard.at[model, 'Matches'] / beta) for model in leaderboard.index]
+    weights = weights / np.sum(weights) # Normalize weights
+    selected = np.random.choice(leaderboard.index, 2, replace=False, p=weights)
+    np.random.shuffle(selected)
+    model1, model2 = selected
+    return model1, model2
+
+async def simulate(iter : int, beta : int, criteria : str) -> pd.DataFrame:
+    """
+    Simulate matches between random models.
+
+    Args:
+        iter (int): The number of matches to simulate.
+        beta (int): The damping factor for the Elo update.
+        criteria (str): The criteria for the rating.
+
+    Returns:
+        leaderboard (pd.DataFrame): Updated leaderboard after simulation
+    """
+    data = await generate_data()
+
+    leaderboard = await generate_leaderboard(criteria)
+    leaderboard.set_index('Model', inplace=True)
+
+    for _ in range(iter):
+        # Generate random matchups
+        timestamp = time.time()
+        model1, model2 = generate_matchup(leaderboard, beta)
+        R1, R2 = leaderboard.at[model1, 'Elo'], leaderboard.at[model2, 'Elo']
+        R1_new, R2_new = update_ratings(R1, R2)
+
+        # Update leaderboard
+        leaderboard.at[model1, 'Elo'], leaderboard.at[model2, 'Elo'] = R1_new, R2_new
+        leaderboard.at[model1, 'Wins'] += 1
+        leaderboard.at[model1, 'Matches'] += 1
+        leaderboard.at[model2, 'Matches'] += 1
+        leaderboard.at[model1, 'Win Rate'] = np.round(leaderboard.at[model1, 'Wins'] / leaderboard.at[model1, 'Matches'], 2)
+        leaderboard.at[model2, 'Win Rate'] = np.round(leaderboard.at[model2, 'Wins'] / leaderboard.at[model2, 'Matches'], 2)
+
+        # Save match data
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': model1, 
+            'Opponent': model2, 
+            'Won': True, 
+            'Elo': leaderboard.at[model1, 'Elo'], 
+            'Win Rate': leaderboard.at[model1, 'Win Rate'],
+            'Matches': leaderboard.at[model1, 'Matches'],
+            'Timestamp': timestamp,
+            'UUID': None
+        }
+
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': model2, 
+            'Opponent': model1, 
+            'Won': False, 
+            'Elo': leaderboard.at[model2, 'Elo'], 
+            'Win Rate': leaderboard.at[model2, 'Win Rate'],
+            'Matches': leaderboard.at[model2, 'Matches'],
+            'Timestamp': timestamp,
+            'UUID': None
+        }
+
+    leaderboard = leaderboard.sort_values('Elo', ascending=False).reset_index(drop=False)
+
+    await asyncio.gather(
+        write_to_s3(f'leaderboard_{criteria}.csv', leaderboard), 
+        write_to_s3('data.csv', data)
+    )
+
+    return leaderboard
+
+
+async def submit_rating(criteria : str, video : str, winner : str, loser : str, uuid : str) -> None:
+    """
+    Submit a rating for a match.
+
+    Args:
+        criteria (str): The criteria for the rating.
+        winner (str): The winning model.
+        loser (str): The losing model.
+        uuid (str): The UUID of the session.
+    """
+    async with submit_lock:
+        data = await generate_data()
+
+        leaderboard = await generate_leaderboard(criteria)
+        leaderboard.set_index('Model', inplace=True)
+
+        if winner is None or loser is None or video is None:
+            return leaderboard
+
+        timestamp = time.time()
+        R_win, R_lose = leaderboard.at[winner, 'Elo'], leaderboard.at[loser, 'Elo']
+        R_win_new, R_lose_new = update_ratings(R_win, R_lose)
+
+        # Update leaderboard
+        leaderboard.loc[[winner, loser], 'Elo'] = [R_win_new, R_lose_new]
+        leaderboard.at[winner, 'Wins'] += 1
+        leaderboard.loc[[winner, loser], 'Matches'] += [1, 1]
+        leaderboard.loc[[winner, loser], 'Win Rate'] = (
+            leaderboard.loc[[winner, loser], 'Wins'] / leaderboard.loc[[winner, loser], 'Matches']
+        ).apply(lambda x: round(x, 2))
+
+        # Save match data
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': winner, 
+            'Opponent': loser, 
+            'Won': True, 
+            'Elo': leaderboard.at[winner, 'Elo'], 
+            'Win Rate': leaderboard.at[winner, 'Win Rate'],
+            'Matches': leaderboard.at[winner, 'Matches'],
+            'Video': video,
+            'Timestamp': timestamp,
+            'UUID': uuid
+        }
+        
+        data.loc[len(data)] = {
+            'Criteria': criteria,
+            'Model': loser, 
+            'Opponent': winner, 
+            'Won': False, 
+            'Elo': leaderboard.at[loser, 'Elo'], 
+            'Win Rate': leaderboard.at[loser, 'Win Rate'],
+            'Matches': leaderboard.at[loser, 'Matches'],
+            'Video': video,
+            'Timestamp': timestamp,
+            'UUID': uuid
+        }
+
+        leaderboard = leaderboard.sort_values('Elo', ascending=False).reset_index(drop=False)
+        await asyncio.gather(
+            write_to_s3(f'leaderboard_{criteria}.csv', leaderboard),
+            write_to_s3('data.csv', data)
+        )
+        return leaderboard