Temp commit

utils.py

@@ -1,25 +1,23 @@
-import os
-import math
-import time
-import json
-import wandb
-import pickle
-import datetime
 import argparse
+import datetime
 import functools
+import json
+import math
+import os
+import time
 import traceback
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
 
-import pandas as pd
-import numpy as np
 import bittensor as bt
-
-from dotenv import load_dotenv
-from dataclasses import dataclass
-from typing import Dict, List, Any, Optional, Tuple
+import numpy as np
+import pandas as pd
+import wandb
 from bittensor.extrinsics.serving import get_metadata
+from dotenv import load_dotenv
 
-
-NETUID = 9
+# TODO: Update once registered
+NETUID = 179
 DELAY_SECS = 3
 RETRIES = 3
 
@@ -27,18 +25,22 @@ load_dotenv()
 
 WANDB_TOKEN = os.environ.get("WANDB_API_KEY", None)
 SUBTENSOR_ENDPOINT = os.environ.get("SUBTENSOR_ENDPOINT", None)
-VALIDATOR_WANDB_PROJECT = "opentensor-dev/pretraining-subnet"
-BENCHMARK_WANDB_PROJECT = "pretraining-benchmark-data"
+VALIDATOR_WANDB_PROJECT = "rusticluftig/finetuning"
+BENCHMARK_WANDB_PROJECT = ""
 BENCHMARK_FLAG = os.environ.get("BENCHMARK_FLAG", None)
 
-@dataclass
+
+@dataclass(frozen=True)
 class ModelData:
     uid: int
     hotkey: str
+    competition_id: int
     namespace: str
     name: str
     commit: str
-    hash: str
+
+    # Hash of (hash(model) + hotkey)
+    secure_hash: str
     block: int
     incentive: float
     emission: float
@@ -60,8 +62,9 @@ class ModelData:
             hotkey=hotkey,
             namespace=tokens[0],
             name=tokens[1],
-            commit=tokens[2] if tokens[2] != "None" else None,
-            hash=tokens[3] if tokens[3] != "None" else None,
+            commit=tokens[2],
+            secure_hash=tokens[3],
+            competition_id=int(tokens[4]),
             block=block,
             incentive=incentive,
             emission=emission,
@@ -69,6 +72,7 @@ class ModelData:
 
 
 def run_with_retries(func, *args, **kwargs):
+    """Runs a provided function with retries in the event of a failure."""
     for i in range(0, RETRIES):
         try:
             return func(*args, **kwargs)
@@ -81,12 +85,18 @@ def run_with_retries(func, *args, **kwargs):
 
 
 def get_subtensor_and_metagraph() -> Tuple[bt.subtensor, bt.metagraph]:
+    """Returns a subtensor and metagraph for the finetuning subnet."""
 
     def _internal() -> Tuple[bt.subtensor, bt.metagraph]:
         if SUBTENSOR_ENDPOINT:
             parser = argparse.ArgumentParser()
             bt.subtensor.add_args(parser)
-            subtensor = bt.subtensor(config=bt.config(parser=parser, args=["--subtensor.chain_endpoint", SUBTENSOR_ENDPOINT]))
+            subtensor = bt.subtensor(
+                config=bt.config(
+                    parser=parser,
+                    args=["--subtensor.chain_endpoint", SUBTENSOR_ENDPOINT],
+                )
+            )
         else:
             subtensor = bt.subtensor("finney")
 
@@ -235,7 +245,11 @@ def get_losses_over_time(wandb_runs: List) -> pd.DataFrame:
         for _, uid_data in all_uid_data.items():
             loss = uid_data.get("average_loss", math.inf)
             # Filter out the numbers from the exploit and when validators lost the best model.
-            if loss < best_loss and (loss > 2.5 or timestamp > datetime.datetime(2024,2,12)) and (loss < 5 or timestamp > datetime.datetime(2024,3,27)):
+            if (
+                loss < best_loss
+                and (loss > 2.5 or timestamp > datetime.datetime(2024, 2, 12))
+                and (loss < 5 or timestamp > datetime.datetime(2024, 3, 27))
+            ):
                 best_loss = uid_data["average_loss"]
         if best_loss != math.inf:
             timestamps.append(timestamp)
@@ -298,53 +312,67 @@ def get_benchmarks() -> Tuple[pd.DataFrame, datetime.datetime]:
         if artifacts:
             table = artifacts[-1].get("benchmarks")
             if table:
-                return table.get_dataframe(), datetime.datetime.strptime(run.metadata["startedAt"], "%Y-%m-%dT%H:%M:%S.%f")
+                return table.get_dataframe(), datetime.datetime.strptime(
+                    run.metadata["startedAt"], "%Y-%m-%dT%H:%M:%S.%f"
+                )
     bt.logging.error("Failed to get benchmarks from Wandb.")
     return None, None
 
 
-def make_validator_dataframe(validator_df: pd.DataFrame, model_data: ModelData) -> pd.DataFrame:
+def make_validator_dataframe(
+    validator_df: pd.DataFrame, model_data: ModelData
+) -> pd.DataFrame:
 
     values = [
-            [uid, int(validator_df[uid][1]), round(validator_df[uid][0], 4)]
-            + [
-                validator_df[uid][-1].get(c.uid)
-                for c in model_data
-                if c.incentive
-            ]
-            for uid, _ in sorted(
-                zip(
-                    validator_df.keys(),
-                    [validator_df[x][1] for x in validator_df.keys()],
-                ),
-                key=lambda x: x[1],
-                reverse=True,
-            )
-        ]
-    dtypes = {"UID":int, "Stake (τ)":float, "V-Trust":float}
-    dtypes.update({
+        [uid, int(validator_df[uid][1]), round(validator_df[uid][0], 4)]
+        + [validator_df[uid][-1].get(c.uid) for c in model_data if c.incentive]
+        for uid, _ in sorted(
+            zip(
+                validator_df.keys(),
+                [validator_df[x][1] for x in validator_df.keys()],
+            ),
+            key=lambda x: x[1],
+            reverse=True,
+        )
+    ]
+    dtypes = {"UID": int, "Stake (τ)": float, "V-Trust": float}
+    dtypes.update(
+        {
             f"{c.namespace}/{c.name} ({c.commit[0:8]})": float
             for c in model_data
             if c.incentive
-        })
+        }
+    )
     return pd.DataFrame(values, columns=dtypes.keys()).astype(dtypes)
 
+
 def make_metagraph_dataframe(metagraph: bt.metagraph, weights=False) -> pd.DataFrame:
 
-    cols = ['stake','emission','trust','validator_trust','dividends','incentive','R', 'consensus','validator_permit']
+    cols = [
+        "stake",
+        "emission",
+        "trust",
+        "validator_trust",
+        "dividends",
+        "incentive",
+        "R",
+        "consensus",
+        "validator_permit",
+    ]
 
     frame = pd.DataFrame({k: getattr(metagraph, k) for k in cols})
-    frame['block'] = metagraph.block.item()
-    frame['netuid'] = NETUID
-    frame['uid'] = range(len(frame))
-    frame['hotkey'] = [axon.hotkey for axon in metagraph.axons]
-    frame['coldkey'] = [axon.coldkey for axon in metagraph.axons]
+    frame["block"] = metagraph.block.item()
+    frame["netuid"] = NETUID
+    frame["uid"] = range(len(frame))
+    frame["hotkey"] = [axon.hotkey for axon in metagraph.axons]
+    frame["coldkey"] = [axon.coldkey for axon in metagraph.axons]
     if weights and metagraph.W is not None:
         # convert NxN tensor to a list of lists so it fits into the dataframe
-        frame['weights'] = [w.tolist() for w in metagraph.W]
+        frame["weights"] = [w.tolist() for w in metagraph.W]
 
     return frame
 
+
 def load_state_vars() -> dict[Any]:
     while True:
         try:
@@ -355,8 +383,11 @@ def load_state_vars() -> dict[Any]:
             model_data: List[ModelData] = get_subnet_data(subtensor, metagraph)
             model_data.sort(key=lambda x: x.incentive, reverse=True)
 
-            bt.logging.success(f'Loaded {len(model_data)} models')
-            vali_runs = get_wandb_runs(project=VALIDATOR_WANDB_PROJECT, filters={"config.type": "validator", "config.uid": 238})
+            bt.logging.success(f"Loaded {len(model_data)} models")
+            vali_runs = get_wandb_runs(
+                project=VALIDATOR_WANDB_PROJECT,
+                filters={"config.type": "validator", "config.uid": 238},
+            )
 
             scores = get_scores([x.uid for x in model_data], vali_runs)
 
@@ -385,40 +416,98 @@ def load_state_vars() -> dict[Any]:
             time.sleep(30)
 
     return {
-        'metagraph': metagraph,
+        "metagraph": metagraph,
         "model_data": model_data,
         "vali_runs": vali_runs,
         "scores": scores,
         "validator_df": validator_df,
         "benchmarks": benchmarks,
-        "benchmark_timestamp": benchmark_timestamp
+        "benchmark_timestamp": benchmark_timestamp,
     }
 
+
 def test_load_state_vars():
 
     subtensor = bt.subtensor("finney")
     metagraph = subtensor.metagraph(NETUID, lite=True)
     model_data = [
-        ModelData(uid=253, hotkey='5DjoPAgZ54Zf6NsuiVYh8RjonnWWWREE2iXBNzM2VDBMQDPm', namespace='jw-hf-test', name='jw2', commit='aad131f6b02219964e6dcf749c2a23e75a7ceca8', hash='L1ImYzWJwV+9KSnZ2TYW0Iy2KMcVjJVTd30YJoRkpbw=', block=3131103, incentive=1.0, emission=209.06051635742188),
-        ModelData(uid=1, hotkey='5CccVtjk4yamCao6QYgEg7jc8vktdj16RbLKNUftHfEsjuJS', namespace='borggAI', name='bittensor-subnet9-models', commit='d373864bc6c972872edb8db95eed570958054bac', hash='+drdTIKYEGYClW2FFVVID6A2Dh//4rLmExRFCJsH6Y4=', block=2081837, incentive=0.0, emission=0.0),
-        ModelData(uid=2, hotkey='5HYwoXaczs3jAptbb5mk4aUCkgZqeNcNzJKxSec97GwasfLy', namespace='jungiebeen', name='pretrain1', commit='4c0c6bfd0f92e243d6c8a82209142e7204c852c3', hash='ld/agc0XIWICom/Cpj0fkQLcMogMNj/F65MJogK5RLY=', block=2467482, incentive=0.0, emission=0.0),
-        ModelData(uid=3, hotkey='5Dnb6edh9yTeEp5aasRPZVPRAkxvQ6qnERVcXw22awMZ5rxm', namespace='jungiebeen', name='pretrain2', commit='e827b7281c92224adb11124489cc45356553a87a', hash='ld/agc0XIWICom/Cpj0fkQLcMogMNj/F65MJogK5RLY=', block=2467497, incentive=0.0, emission=0.0),
-        ModelData(uid=4, hotkey='5FRfca8NbnH424WaX43PMhKBnbLA1bZpRRoXXiVs6HgsxN4K', namespace='ZainAli60', name='mine_modeles', commit='8a4ed4ad1f1fb58d424fd22e8e9874b87d32917c', hash='tVcbZAFoNIOF+Ntxq31OQ2NrLXf5iFCmmPUJlpkMYYo=', block=2508509, incentive=0.0, emission=0.0)
+        ModelData(
+            uid=253,
+            hotkey="5DjoPAgZ54Zf6NsuiVYh8RjonnWWWREE2iXBNzM2VDBMQDPm",
+            namespace="jw-hf-test",
+            name="jw2",
+            commit="aad131f6b02219964e6dcf749c2a23e75a7ceca8",
+            secure_hash="L1ImYzWJwV+9KSnZ2TYW0Iy2KMcVjJVTd30YJoRkpbw=",
+            block=3131103,
+            incentive=1.0,
+            emission=209.06051635742188,
+        ),
+        ModelData(
+            uid=1,
+            hotkey="5CccVtjk4yamCao6QYgEg7jc8vktdj16RbLKNUftHfEsjuJS",
+            namespace="borggAI",
+            name="bittensor-subnet9-models",
+            commit="d373864bc6c972872edb8db95eed570958054bac",
+            secure_hash="+drdTIKYEGYClW2FFVVID6A2Dh//4rLmExRFCJsH6Y4=",
+            block=2081837,
+            incentive=0.0,
+            emission=0.0,
+        ),
+        ModelData(
+            uid=2,
+            hotkey="5HYwoXaczs3jAptbb5mk4aUCkgZqeNcNzJKxSec97GwasfLy",
+            namespace="jungiebeen",
+            name="pretrain1",
+            commit="4c0c6bfd0f92e243d6c8a82209142e7204c852c3",
+            secure_hash="ld/agc0XIWICom/Cpj0fkQLcMogMNj/F65MJogK5RLY=",
+            block=2467482,
+            incentive=0.0,
+            emission=0.0,
+        ),
+        ModelData(
+            uid=3,
+            hotkey="5Dnb6edh9yTeEp5aasRPZVPRAkxvQ6qnERVcXw22awMZ5rxm",
+            namespace="jungiebeen",
+            name="pretrain2",
+            commit="e827b7281c92224adb11124489cc45356553a87a",
+            secure_hash="ld/agc0XIWICom/Cpj0fkQLcMogMNj/F65MJogK5RLY=",
+            block=2467497,
+            incentive=0.0,
+            emission=0.0,
+        ),
+        ModelData(
+            uid=4,
+            hotkey="5FRfca8NbnH424WaX43PMhKBnbLA1bZpRRoXXiVs6HgsxN4K",
+            namespace="ZainAli60",
+            name="mine_modeles",
+            commit="8a4ed4ad1f1fb58d424fd22e8e9874b87d32917c",
+            secure_hash="tVcbZAFoNIOF+Ntxq31OQ2NrLXf5iFCmmPUJlpkMYYo=",
+            block=2508509,
+            incentive=0.0,
+            emission=0.0,
+        ),
     ]
-    vali_runs = get_wandb_runs(project=VALIDATOR_WANDB_PROJECT, filters={"config.type": "validator", "config.uid": 238})
+    vali_runs = get_wandb_runs(
+        project=VALIDATOR_WANDB_PROJECT,
+        filters={"config.type": "validator", "config.uid": 238},
+    )
 
     scores = get_scores([x.uid for x in model_data], vali_runs)
 
     validator_df = {
         28: (1.0, 33273.4453125, {253: 1.0}),
-        49: (0.9127794504165649,
-        10401.677734375,
-        {7: 0.0867,
-        217: 0.0001,
-        219: 0.0001,
-        241: 0.0001,
-        248: 0.0001,
-        253: 0.9128}),
+        49: (
+            0.9127794504165649,
+            10401.677734375,
+            {
+                7: 0.0867,
+                217: 0.0001,
+                219: 0.0001,
+                241: 0.0001,
+                248: 0.0001,
+                253: 0.9128,
+            },
+        ),
         78: (1.0, 26730.37109375, {253: 1.0}),
         116: (1.0, 629248.4375, {253: 1.0}),
         150: (1.0, 272634.53125, {253: 1.0}),
@@ -438,11 +527,11 @@ def test_load_state_vars():
         249: (1.0, 478127.3125, {253: 1.0}),
         252: (1.0, 442395.03125, {253: 1.0}),
         254: (1.0, 46845.2109375, {253: 1.0}),
-        255: (1.0, 28977.56640625, {253: 1.0})
+        255: (1.0, 28977.56640625, {253: 1.0}),
     }
 
     return {
-        'metagraph': metagraph,
+        "metagraph": metagraph,
         "model_data": model_data,
         "vali_runs": vali_runs,
         "scores": scores,