Working starting point

app.py

@@ -1,6 +1,6 @@
 import gradio as gr
 import bittensor as bt
-import typing
+from typing import Dict, List, Any, Optional, Tuple
 from bittensor.extrinsics.serving import get_metadata
 from dataclasses import dataclass
 import requests
@@ -9,17 +9,21 @@ import math
 import os
 import datetime
 import time
+import json
+import pandas as pd
 from dotenv import load_dotenv
 from huggingface_hub import HfApi
 from apscheduler.schedulers.background import BackgroundScheduler
 
 load_dotenv()
 
-FONT = """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
-TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 6 Leaderboard</h1>"""
-#IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
-HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/RaoFoundation/pretraining" target="_blank">Subnet 9</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that rewards miners for producing pretrained Foundation-Models on the <a href="https://huggingface.co/datasets/tiiuae/falcon-refinedweb" target="_blank">Falcon Refined Web dataset</a>. It acts like a continuous benchmark whereby miners are rewarded for attaining the best losses on randomly sampled pages of Falcon. The models with the best head-to-head loss on the evaluation data receive a steady emission of TAO.</h3>"""
-EVALUATION_DETAILS = """<b>Name</b> is the 🤗 Hugging Face model name (click to go to the model card). <b>Rewards / Day</b> are the expected rewards per day for each model. <b>Last Average Loss</b> is the last loss value on the evaluation data for the model as calculated by a validator (lower is better). <b>UID</b> is the Bittensor user id of the submitter. <b>Block</b> is the Bittensor block that the model was submitted in. More stats on <a href="https://taostats.io/subnets/netuid-6/" target="_blank">taostats</a>."""
+FONT = (
+    """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
+)
+TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 9 Leaderboard</h1>"""
+# IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
+HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/RaoFoundation/pretraining" target="_blank">Subnet 9</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that rewards miners for producing pretrained Foundation-Models on the <a href="https://huggingface.co/datasets/tiiuae/falcon-refinedweb" target="_blank">Falcon Refined Web dataset</a>. It acts like a continuous benchmark whereby miners are rewarded for attaining the best losses on randomly sampled pages of Falcon.<br/>The models with the best head-to-head loss on the evaluation data receive a steady emission of TAO.</h3>"""
+EVALUATION_DETAILS = """<ul><li><b>Name:</b> the 🤗 Hugging Face model name (click to go to the model card)</li><li><b>Rewards / Day:</b> the expected rewards per day based on current ranking.</li><li><b>Last Average Loss:</b> the last loss value on the evaluation data for the model as calculated by a validator (lower is better)</li><li><b>UID:</b> the Bittensor UID of the miner</li><li><b>Block:</b> the Bittensor block that the model was submitted in</li></ul><br/>More stats on <a href="https://taostats.io/subnets/netuid-9/" target="_blank">taostats</a>."""
 EVALUATION_HEADER = """<h3 align="center">Shows the latest internal evaluation statistics as calculated by the Opentensor validator</h3>"""
 VALIDATOR_WANDB_PROJECT = "opentensor-dev/pretraining-subnet"
 H4_TOKEN = os.environ.get("H4_TOKEN", None)
@@ -30,10 +34,9 @@ MAX_AVG_LOSS_POINTS = 1
 RETRIES = 5
 DELAY_SECS = 3
 NETUID = 9
-# TODO: Update this for SN 9.
-SUBNET_START_BLOCK = 2225782
 SECONDS_PER_BLOCK = 12
 
+
 @dataclass
 class ModelData:
     uid: int
@@ -47,7 +50,15 @@ class ModelData:
     emission: float
 
     @classmethod
-    def from_compressed_str(cls, uid: int, hotkey: str, cs: str, block: int, incentive: float, emission: float):
+    def from_compressed_str(
+        cls,
+        uid: int,
+        hotkey: str,
+        cs: str,
+        block: int,
+        incentive: float,
+        emission: float,
+    ):
         """Returns an instance of this class from a compressed string representation"""
         tokens = cs.split(":")
         return ModelData(
@@ -59,9 +70,10 @@ class ModelData:
             hash=tokens[3] if tokens[3] != "None" else None,
             block=block,
             incentive=incentive,
-            emission=emission
+            emission=emission,
         )
 
+
 def run_with_retries(func, *args, **kwargs):
     for i in range(0, RETRIES):
         try:
@@ -72,17 +84,30 @@ def run_with_retries(func, *args, **kwargs):
             time.sleep(DELAY_SECS)
     raise RuntimeError("Should never happen")
 
-def get_subtensor_and_metagraph() -> typing.Tuple[bt.subtensor, bt.metagraph]:
-    def _internal() -> typing.Tuple[bt.subtensor, bt.metagraph]:
-        subtensor: bt.subtensor = bt.subtensor("finney")
-        metagraph: bt.metagraph = bt.metagraph(9, lite=False)
+
+def get_subtensor_and_metagraph() -> Tuple[bt.subtensor, bt.metagraph]:
+    def _internal() -> Tuple[bt.subtensor, bt.metagraph]:
+        subtensor = bt.subtensor("finney")
+        metagraph = bt.metagraph(NETUID, lite=False)
         return subtensor, metagraph
+
     return run_with_retries(_internal)
 
+
 def get_tao_price() -> float:
-    return run_with_retries(lambda: float(requests.get("https://api.kucoin.com/api/v1/market/stats?symbol=TAO-USDT").json()["data"]["last"]))
+    return run_with_retries(
+        lambda: float(
+            requests.get(
+                "https://api.kucoin.com/api/v1/market/stats?symbol=TAO-USDT"
+            ).json()["data"]["last"]
+        )
+    )
+
 
-def get_validator_weights(metagraph: bt.metagraph) -> typing.Dict[int, typing.Tuple[float, int, typing.Dict[int, float]]]:
+def get_validator_weights(
+    metagraph: bt.metagraph,
+) -> Dict[int, Tuple[float, int, Dict[int, float]]]:
+    """Returns a dictionary of validator UIDs to (vtrust, stake, {uid: weight})."""
     ret = {}
     for uid in metagraph.uids.tolist():
         vtrust = metagraph.validator_trust[uid].item()
@@ -96,7 +121,10 @@ def get_validator_weights(metagraph: bt.metagraph) -> typing.Dict[int, typing.Tu
                     ret[uid][-1][ouid] = weight
     return ret
 
-def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
+
+def get_subnet_data(
+    subtensor: bt.subtensor, metagraph: bt.metagraph
+) -> List[ModelData]:
     result = []
     for uid in metagraph.uids.tolist():
         hotkey = metagraph.hotkeys[uid]
@@ -109,62 +137,75 @@ def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.
         chain_str = bytes.fromhex(hex_data).decode()
         block = metadata["block"]
         incentive = metagraph.incentive[uid].nan_to_num().item()
-        emission = metagraph.emission[uid].nan_to_num().item() * 20 # convert to daily TAO
+        emission = (
+            metagraph.emission[uid].nan_to_num().item() * 20
+        )  # convert to daily TAO
 
         model_data = None
         try:
-            model_data = ModelData.from_compressed_str(uid, hotkey, chain_str, block, incentive, emission)    
+            model_data = ModelData.from_compressed_str(
+                uid, hotkey, chain_str, block, incentive, emission
+            )
         except:
             continue
 
         result.append(model_data)
     return result
 
+
 def is_floatable(x) -> bool:
-    return (isinstance(x, float) and not math.isnan(x) and not math.isinf(x)) or isinstance(x, int)
-    
-def get_float_score(key: str, history) -> typing.Tuple[typing.Optional[float], bool]:
-    if key in history:
-        data = list(history[key])
-        if len(data) > 0:
-            if is_floatable(data[-1]):
-                return float(data[-1]), True
-            else:
-                data = [float(x) for x in data if is_floatable(x)]
-                if len(data) > 0:
-                    return float(data[-1]), False
-    return None, False
-
-def get_scores(uids: typing.List[int]) -> typing.Dict[int, typing.Dict[str, typing.Optional[float]]]:
+    return (
+        isinstance(x, float) and not math.isnan(x) and not math.isinf(x)
+    ) or isinstance(x, int)
+
+
+def get_scores(
+    uids: List[int],
+) -> Dict[int, Dict[str, Optional[float]]]:
     api = wandb.Api(api_key=WANDB_TOKEN)
-    runs = list(api.runs(VALIDATOR_WANDB_PROJECT),
-                filters={
-                    "type": "validator",
-                    "uid": 238
-                })
+    runs = list(
+        api.runs(
+            VALIDATOR_WANDB_PROJECT,
+            filters={"config.type": "validator", "config.uid": 238},
+        )
+    )
 
     result = {}
-    for run in runs:
-        history = run.history()
+    previous_timestamp = None
+    # Iterate through the runs until we've processed all the uids.
+    for i, run in enumerate(runs):
+        if not "original_format_json" in run.summary:
+            continue
+        data = json.loads(run.summary["original_format_json"])
+        all_uid_data = data["uid_data"]
+        timestamp = data["timestamp"]
+
+        # Make sure runs are indeed in descending time order.
+        assert (
+            previous_timestamp is None or timestamp < previous_timestamp
+        ), f"Timestamps are not in descending order: {timestamp} >= {previous_timestamp}"
+        previous_timestamp = timestamp
+
         for uid in uids:
             if uid in result:
                 continue
-            avg_loss, avg_loss_fresh = get_float_score(f"uid_data.{uid}", history)
-            win_rate, win_rate_fresh = get_float_score(f"win_rate_data.{uid}", history)
-            win_total, win_total_fresh = get_float_score(f"win_total_data.{uid}", history)
-            weight, weight_fresh = get_float_score(f"weight_data.{uid}", history)
-            result[uid] = {
-                "avg_loss": avg_loss,
-                "win_rate": win_rate,
-                "win_total": win_total,
-                "weight": weight,
-                "fresh": avg_loss_fresh and win_rate_fresh and win_total_fresh
-            }
-        if len(result.keys()) == len(uids):
+            if str(uid) in all_uid_data:
+                uid_data = all_uid_data[str(uid)]
+                # Only the most recent run is fresh.
+                is_fresh = i == 0
+                result[uid] = {
+                    "avg_loss": uid_data.get("average_loss", None),
+                    "win_rate": uid_data.get("win_rate", None),
+                    "win_total": uid_data.get("win_total", None),
+                    "weight": uid_data.get("weight", None),
+                    "fresh": is_fresh,
+                }
+        if len(result) == len(uids):
             break
     return result
 
-def format_score(uid: int, scores, key) -> typing.Optional[float]:
+
+def format_score(uid: int, scores, key) -> Optional[float]:
     if uid in scores:
         if key in scores[uid]:
             point = scores[uid][key]
@@ -172,102 +213,129 @@ def format_score(uid: int, scores, key) -> typing.Optional[float]:
                 return round(scores[uid][key], 4)
     return None
 
-def next_tempo(start_block: int, tempo: int, block: int) -> int:
-    start_num = start_block + tempo
-    intervals = (block - start_num) // tempo
-    nearest_num = start_num + ((intervals + 1) * tempo)
-    return nearest_num
+
+def next_epoch(subtensor: bt.subtensor, block: int) -> int:
+    return subtensor.get_subnet_hyperparameters(
+        NETUID
+    ).tempo - subtensor.blocks_since_epoch(NETUID, block)
+
 
 def get_next_update_div(current_block: int, next_update_block: int) -> str:
     now = datetime.datetime.now()
     blocks_to_go = next_update_block - current_block
-    next_update_time = now + datetime.timedelta(seconds=blocks_to_go * SECONDS_PER_BLOCK)
+    next_update_time = now + datetime.timedelta(
+        seconds=blocks_to_go * SECONDS_PER_BLOCK
+    )
     delta = next_update_time - now
     return f"""<div align="center" style="font-size: larger;">Next reward update: <b>{blocks_to_go}</b> blocks (~{int(delta.total_seconds() // 60)} minutes)</div>"""
 
-subtensor, metagraph = get_subtensor_and_metagraph()
-
-tao_price = get_tao_price()
-
-leaderboard_df = get_subnet_data(subtensor, metagraph)
-leaderboard_df.sort(key=lambda x: x.incentive, reverse=True)
 
-scores = get_scores([x.uid for x in leaderboard_df])
-
-current_block = metagraph.block.item()
-next_update = next_tempo(
-    SUBNET_START_BLOCK,
-    subtensor.get_subnet_hyperparameters(NETUID).tempo,
-    current_block
-)
-
-validator_df = get_validator_weights(metagraph)
-weight_keys = set()
-for uid, stats in validator_df.items():
-    weight_keys.update(stats[-1].keys())
-
-def leaderboard_data(show_stale: bool):
-    value = [
+def leaderboard_data(
+    leaderboard: List[ModelData],
+    scores: Dict[int, Dict[str, Optional[float]]],
+    show_stale: bool,
+) -> List[List[Any]]:
+    """Returns the leaderboard data, based on models data and UID scores."""
+    return [
         [
-            f'[{c.namespace}/{c.name} ({c.commit[0:8]})](https://huggingface.co/{c.namespace}/{c.name}/commit/{c.commit})',
+            f"[{c.namespace}/{c.name} ({c.commit[0:8]})](https://huggingface.co/{c.namespace}/{c.name}/commit/{c.commit})",
             format_score(c.uid, scores, "win_rate"),
             format_score(c.uid, scores, "avg_loss"),
             format_score(c.uid, scores, "weight"),
             c.uid,
-            c.block
-        ] for c in leaderboard_df if scores[c.uid]["fresh"] or show_stale
+            c.block,
+        ]
+        for c in leaderboard
+        if (c.uid in scores and scores[c.uid]["fresh"]) or show_stale
     ]
-    return value
 
-demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
-with demo:
-    gr.HTML(FONT)
-    gr.HTML(TITLE)
-    #gr.HTML(IMAGE)
-    gr.HTML(HEADER)
+def restart_space():
+    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
-    gr.HTML(value=get_next_update_div(current_block, next_update))
 
-    gr.Label(
-        value={ f"{c.namespace}/{c.name} ({c.commit[0:8]}) · ${round(c.emission * tao_price, 2):,} (τ{round(c.emission, 2):,})": c.incentive for c in leaderboard_df if c.incentive},
-        num_top_classes=10,
-    )
-    
-    with gr.Accordion("Evaluation Stats"):
-        gr.HTML(EVALUATION_HEADER)
-        show_stale = gr.Checkbox(label="Show Stale", interactive=True)
-        leaderboard_table = gr.components.Dataframe(
-            value=leaderboard_data(show_stale.value),
-            headers=["Name", "Win Rate", "Average Loss", "Weight", "UID", "Block"],
-            datatype=["markdown", "number", "number", "number", "number", "number"],
-            elem_id="leaderboard-table",
-            interactive=False,
-            visible=True,
-        )
-        gr.HTML(EVALUATION_DETAILS)
-        show_stale.change(leaderboard_data, [show_stale], leaderboard_table)
-
-    with gr.Accordion("Validator Stats"):
-        validator_table = gr.components.Dataframe(
-            value=[
-                [uid, int(validator_df[uid][1]), round(validator_df[uid][0], 4)] + [validator_df[uid][-1].get(c.uid) for c in leaderboard_df if c.incentive]
-                for uid, _ in sorted(
-                        zip(validator_df.keys(), [validator_df[x][1] for x in validator_df.keys()]),
-                        key=lambda x: x[1],
-                        reverse=True
-                    )
-            ],
-            headers=["UID", "Stake (τ)", "V-Trust"] + [f"{c.namespace}/{c.name} ({c.commit[0:8]})" for c in leaderboard_df if c.incentive],
-            datatype=["number", "number", "number"] + ["number" for c in leaderboard_df if c.incentive],
-            interactive=False,
-            visible=True,
-        )
+def main():
+    subtensor, metagraph = get_subtensor_and_metagraph()
 
-def restart_space():
-    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
+    tao_price = get_tao_price()
+
+    model_data: List[ModelData] = get_subnet_data(subtensor, metagraph)
+    model_data.sort(key=lambda x: x.incentive, reverse=True)
+
+    scores = get_scores([x.uid for x in model_data])
+
+    current_block = metagraph.block.item()
+    next_epoch_block = next_epoch(subtensor, current_block)
+
+    validator_df = get_validator_weights(metagraph)
+    weight_keys = set()
+    for uid, stats in validator_df.items():
+        weight_keys.update(stats[-1].keys())
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=60 * 15) # restart every 15 minutes
-scheduler.start()
+    demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
+    with demo:
+        gr.HTML(FONT)
+        gr.HTML(TITLE)
+        # gr.HTML(IMAGE)
+        gr.HTML(HEADER)
 
-demo.launch()
+        gr.HTML(value=get_next_update_div(current_block, next_epoch_block))
+
+        gr.Label(
+            value={
+                f"{c.namespace}/{c.name} ({c.commit[0:8]}) · ${round(c.emission * tao_price, 2):,} (τ{round(c.emission, 2):,})": c.incentive
+                for c in model_data
+                if c.incentive
+            },
+            num_top_classes=10,
+        )
+
+        with gr.Accordion("Evaluation Stats"):
+            gr.HTML(EVALUATION_HEADER)
+            show_stale = gr.Checkbox(label="Show Stale", interactive=True)
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_data(model_data, scores, show_stale.value),
+                headers=["Name", "Win Rate", "Average Loss", "Weight", "UID", "Block"],
+                datatype=["markdown", "number", "number", "number", "number", "number"],
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+            gr.HTML(EVALUATION_DETAILS)
+            show_stale.change(lambda stale: leaderboard_data(model_data, scores, stale), inputs=[show_stale], outputs=leaderboard_table)
+
+        with gr.Accordion("Validator Stats"):
+            gr.components.Dataframe(
+                value=[
+                    [uid, int(validator_df[uid][1]), round(validator_df[uid][0], 4)]
+                    + [validator_df[uid][-1].get(c.uid) for c in model_data if c.incentive]
+                    for uid, _ in sorted(
+                        zip(
+                            validator_df.keys(),
+                            [validator_df[x][1] for x in validator_df.keys()],
+                        ),
+                        key=lambda x: x[1],
+                        reverse=True,
+                    )
+                ],
+                headers=["UID", "Stake (τ)", "V-Trust"]
+                + [
+                    f"{c.namespace}/{c.name} ({c.commit[0:8]})"
+                    for c in model_data
+                    if c.incentive
+                ],
+                datatype=["number", "number", "number"]
+                + ["number" for c in model_data if c.incentive],
+                interactive=False,
+                visible=True,
+            )
+
+
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(
+        restart_space, "interval", seconds=60 * 15
+    )  # restart every 15 minutes
+    scheduler.start()
+
+    demo.launch()
+    
+main()

requirements.txt

@@ -3,4 +3,5 @@ requests==2.31.0
 wandb==0.16.2
 python-dotenv==1.0.1
 APScheduler==3.10.1
-huggingface-hub>=0.18.0
+huggingface-hub>=0.18.0
+pandas==2.2.0