Echo commited on
Commit
5bf8c6d
1 Parent(s): 053e485

upgrade bittensor & enhance async fetch subnet 6

Browse files
Files changed (6) hide show
  1. app.py +30 -19
  2. backup/README.md +0 -12
  3. backup/app.py +0 -406
  4. backup/nousgirl.png +0 -0
  5. backup/requirements.txt +0 -7
  6. requirements.txt +2 -2
app.py CHANGED
@@ -15,19 +15,20 @@ from dotenv import load_dotenv
15
  from huggingface_hub import HfApi
16
  from apscheduler.schedulers.background import BackgroundScheduler
17
  from tqdm import tqdm
 
18
 
19
  load_dotenv()
20
 
21
  FONT = """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
22
- TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 6 Leaderboard</h1>"""
23
  IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
24
  HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/NousResearch/finetuning-subnet" target="_blank">Subnet 6</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that incentivizes the creation of the best open models by evaluating submissions on a constant stream of newly generated synthetic GPT-4 data. The models with the best <a href="https://github.com/NousResearch/finetuning-subnet/blob/master/docs/validator.md" target="_blank">head-to-head loss</a> on the evaluation data receive a steady emission of TAO.</h3>"""
25
  EVALUATION_DETAILS = """<b>Name</b> is the 🤗 Hugging Face model name (click to go to the model card). <b>Rewards / Day</b> are the expected rewards per day for each model. <b>Perplexity</b> is represents the loss on all of the evaluation data for the model as calculated by the validator (lower is better). <b>UID</b> is the Bittensor user id of the submitter. <b>Block</b> is the Bittensor block that the model was submitted in. More stats on <a href="https://taostats.io/subnets/netuid-6/" target="_blank">taostats</a>."""
26
- EVALUATION_HEADER = """<h3 align="center">Shows the latest internal evaluation statistics as calculated by a validator run by Nous Research</h3>"""
27
  VALIDATOR_WANDB_PROJECT = os.environ["VALIDATOR_WANDB_PROJECT"]
28
  H4_TOKEN = os.environ.get("H4_TOKEN", None)
29
  API = HfApi(token=H4_TOKEN)
30
- REPO_ID = "NousResearch/finetuning_subnet_leaderboard"
31
  METAGRAPH_RETRIES = 10
32
  METAGRAPH_DELAY_SECS = 30
33
  METADATA_TTL = 10
@@ -43,6 +44,7 @@ class Competition:
43
 
44
  COMPETITIONS = [Competition(id="m1", name="mistral-7b"), Competition(id="g1", name="gemma-2b")]
45
  DEFAULT_COMPETITION_ID = "m1"
 
46
 
47
  def run_in_subprocess(func: functools.partial, ttl: int) -> typing.Any:
48
  """Runs the provided function on a subprocess with 'ttl' seconds to complete.
@@ -160,19 +162,18 @@ def get_validator_weights(metagraph: bt.metagraph) -> typing.Dict[int, typing.Tu
160
  return ret
161
 
162
  def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
163
- result = []
164
- for uid in tqdm(metagraph.uids.tolist(), desc="Metadata for hotkeys"):
 
165
  hotkey = metagraph.hotkeys[uid]
166
  try:
167
- # Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs.
168
  partial = functools.partial(get_metadata, subtensor, metagraph.netuid, hotkey)
169
  metadata = run_in_subprocess(partial, METADATA_TTL)
170
- except KeyboardInterrupt:
171
- raise
172
- except:
173
- metadata = None
174
  if not metadata:
175
- continue
176
 
177
  commitment = metadata["info"]["fields"][0]
178
  hex_data = commitment[list(commitment.keys())[0]][2:]
@@ -181,14 +182,25 @@ def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.
181
  incentive = metagraph.incentive[uid].nan_to_num().item()
182
  emission = metagraph.emission[uid].nan_to_num().item() * 20 # convert to daily TAO
183
 
184
- model_data = None
185
  try:
186
  model_data = ModelData.from_compressed_str(uid, hotkey, chain_str, block, incentive, emission)
187
- except:
188
- continue
 
189
 
190
- result.append(model_data)
191
- return result
 
 
 
 
 
 
 
 
 
 
 
192
 
193
  def floatable(x) -> bool:
194
  return (isinstance(x, float) and not math.isnan(x) and not math.isinf(x)) or isinstance(x, int)
@@ -345,8 +357,7 @@ with demo:
345
  )
346
 
347
  with gr.Accordion("Evaluation Stats"):
348
- gr.HTML(EVALUATION_HEADER)
349
-
350
  with gr.Tabs():
351
  for entry in leaderboard_df:
352
  if entry.competition == competition.id:
@@ -389,7 +400,7 @@ def restart_space():
389
  API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
390
 
391
  scheduler = BackgroundScheduler()
392
- scheduler.add_job(restart_space, "interval", seconds=60 * 15) # restart every 15 minutes
393
  scheduler.start()
394
 
395
  demo.launch()
 
15
  from huggingface_hub import HfApi
16
  from apscheduler.schedulers.background import BackgroundScheduler
17
  from tqdm import tqdm
18
+ import concurrent.futures
19
 
20
  load_dotenv()
21
 
22
  FONT = """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
23
+ TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 6 Leaderboard (with Cortex Foundation validator/subtensor)</h1>"""
24
  IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
25
  HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/NousResearch/finetuning-subnet" target="_blank">Subnet 6</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that incentivizes the creation of the best open models by evaluating submissions on a constant stream of newly generated synthetic GPT-4 data. The models with the best <a href="https://github.com/NousResearch/finetuning-subnet/blob/master/docs/validator.md" target="_blank">head-to-head loss</a> on the evaluation data receive a steady emission of TAO.</h3>"""
26
  EVALUATION_DETAILS = """<b>Name</b> is the 🤗 Hugging Face model name (click to go to the model card). <b>Rewards / Day</b> are the expected rewards per day for each model. <b>Perplexity</b> is represents the loss on all of the evaluation data for the model as calculated by the validator (lower is better). <b>UID</b> is the Bittensor user id of the submitter. <b>Block</b> is the Bittensor block that the model was submitted in. More stats on <a href="https://taostats.io/subnets/netuid-6/" target="_blank">taostats</a>."""
27
+ EVALUATION_HEADER = """<h3 align="center">Shows the latest internal evaluation statistics as calculated by a validator run by Cortex Foundation ({date}) </h3>"""
28
  VALIDATOR_WANDB_PROJECT = os.environ["VALIDATOR_WANDB_PROJECT"]
29
  H4_TOKEN = os.environ.get("H4_TOKEN", None)
30
  API = HfApi(token=H4_TOKEN)
31
+ REPO_ID = "0x9/finetuning_subnet_leaderboard"
32
  METAGRAPH_RETRIES = 10
33
  METAGRAPH_DELAY_SECS = 30
34
  METADATA_TTL = 10
 
44
 
45
  COMPETITIONS = [Competition(id="m1", name="mistral-7b"), Competition(id="g1", name="gemma-2b")]
46
  DEFAULT_COMPETITION_ID = "m1"
47
+ last_refresh = None
48
 
49
  def run_in_subprocess(func: functools.partial, ttl: int) -> typing.Any:
50
  """Runs the provided function on a subprocess with 'ttl' seconds to complete.
 
162
  return ret
163
 
164
  def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
165
+ global last_refresh
166
+ # Function to be executed in a thread
167
+ def fetch_data(uid):
168
  hotkey = metagraph.hotkeys[uid]
169
  try:
 
170
  partial = functools.partial(get_metadata, subtensor, metagraph.netuid, hotkey)
171
  metadata = run_in_subprocess(partial, METADATA_TTL)
172
+ except Exception as e:
173
+ return None
174
+
 
175
  if not metadata:
176
+ return None
177
 
178
  commitment = metadata["info"]["fields"][0]
179
  hex_data = commitment[list(commitment.keys())[0]][2:]
 
182
  incentive = metagraph.incentive[uid].nan_to_num().item()
183
  emission = metagraph.emission[uid].nan_to_num().item() * 20 # convert to daily TAO
184
 
 
185
  try:
186
  model_data = ModelData.from_compressed_str(uid, hotkey, chain_str, block, incentive, emission)
187
+ except Exception as e:
188
+ return None
189
+ return model_data
190
 
191
+ # Use ThreadPoolExecutor to fetch data in parallel
192
+ results = []
193
+ with concurrent.futures.ThreadPoolExecutor() as executor:
194
+ # Prepare the list of futures
195
+ futures = [executor.submit(fetch_data, uid) for uid in metagraph.uids.tolist()]
196
+ for future in tqdm(concurrent.futures.as_completed(futures), desc="Metadata for hotkeys", total=len(futures)):
197
+ result = future.result()
198
+ if result:
199
+ results.append(result)
200
+
201
+
202
+ last_refresh = datetime.datetime.now()
203
+ return results
204
 
205
  def floatable(x) -> bool:
206
  return (isinstance(x, float) and not math.isnan(x) and not math.isinf(x)) or isinstance(x, int)
 
357
  )
358
 
359
  with gr.Accordion("Evaluation Stats"):
360
+ gr.HTML(EVALUATION_HEADER.replace("{date}", last_refresh.strftime("refreshed at %H:%M on %Y-%m-%d")))
 
361
  with gr.Tabs():
362
  for entry in leaderboard_df:
363
  if entry.competition == competition.id:
 
400
  API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
401
 
402
  scheduler = BackgroundScheduler()
403
+ scheduler.add_job(restart_space, "interval", seconds=60*5) # restart every 15 minutes
404
  scheduler.start()
405
 
406
  demo.launch()
backup/README.md DELETED
@@ -1,12 +0,0 @@
1
- ---
2
- title: Finetuning Subnet Leaderboard
3
- emoji: ⚒️
4
- colorFrom: indigo
5
- colorTo: blue
6
- sdk: gradio
7
- sdk_version: 3.41.0
8
- app_file: app.py
9
- pinned: false
10
- ---
11
-
12
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
backup/app.py DELETED
@@ -1,406 +0,0 @@
1
- import gradio as gr
2
- import bittensor as bt
3
- import typing
4
- from bittensor.extrinsics.serving import get_metadata
5
- from dataclasses import dataclass
6
- import requests
7
- import wandb
8
- import math
9
- import os
10
- import datetime
11
- import time
12
- import functools
13
- import multiprocessing
14
- from dotenv import load_dotenv
15
- from huggingface_hub import HfApi
16
- from apscheduler.schedulers.background import BackgroundScheduler
17
- from tqdm import tqdm
18
- import concurrent.futures
19
-
20
- load_dotenv()
21
-
22
- FONT = """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
23
- TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 6 Leaderboard (with Cortex Foundation validator/subtensor)</h1>"""
24
- IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
25
- HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/NousResearch/finetuning-subnet" target="_blank">Subnet 6</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that incentivizes the creation of the best open models by evaluating submissions on a constant stream of newly generated synthetic GPT-4 data. The models with the best <a href="https://github.com/NousResearch/finetuning-subnet/blob/master/docs/validator.md" target="_blank">head-to-head loss</a> on the evaluation data receive a steady emission of TAO.</h3>"""
26
- EVALUATION_DETAILS = """<b>Name</b> is the 🤗 Hugging Face model name (click to go to the model card). <b>Rewards / Day</b> are the expected rewards per day for each model. <b>Perplexity</b> is represents the loss on all of the evaluation data for the model as calculated by the validator (lower is better). <b>UID</b> is the Bittensor user id of the submitter. <b>Block</b> is the Bittensor block that the model was submitted in. More stats on <a href="https://taostats.io/subnets/netuid-6/" target="_blank">taostats</a>."""
27
- EVALUATION_HEADER = """<h3 align="center">Shows the latest internal evaluation statistics as calculated by a validator run by Cortex Foundation ({date}) </h3>"""
28
- VALIDATOR_WANDB_PROJECT = os.environ["VALIDATOR_WANDB_PROJECT"]
29
- H4_TOKEN = os.environ.get("H4_TOKEN", None)
30
- API = HfApi(token=H4_TOKEN)
31
- REPO_ID = "0x9/finetuning_subnet_leaderboard"
32
- METAGRAPH_RETRIES = 10
33
- METAGRAPH_DELAY_SECS = 30
34
- METADATA_TTL = 10
35
- NETUID = 6
36
- SUBNET_START_BLOCK = 2225782
37
- SECONDS_PER_BLOCK = 12
38
- SUBTENSOR = os.environ.get("SUBTENSOR", "finney")
39
-
40
- @dataclass
41
- class Competition:
42
- id: str
43
- name: str
44
-
45
- COMPETITIONS = [Competition(id="m1", name="mistral-7b"), Competition(id="g1", name="gemma-2b")]
46
- DEFAULT_COMPETITION_ID = "m1"
47
- last_refresh = None
48
-
49
- def run_in_subprocess(func: functools.partial, ttl: int) -> typing.Any:
50
- """Runs the provided function on a subprocess with 'ttl' seconds to complete.
51
-
52
- Args:
53
- func (functools.partial): Function to be run.
54
- ttl (int): How long to try for in seconds.
55
-
56
- Returns:
57
- Any: The value returned by 'func'
58
- """
59
-
60
- def wrapped_func(func: functools.partial, queue: multiprocessing.Queue):
61
- try:
62
- result = func()
63
- queue.put(result)
64
- except (Exception, BaseException) as e:
65
- # Catch exceptions here to add them to the queue.
66
- queue.put(e)
67
-
68
- # Use "fork" (the default on all POSIX except macOS), because pickling doesn't seem
69
- # to work on "spawn".
70
- ctx = multiprocessing.get_context("fork")
71
- queue = ctx.Queue()
72
- process = ctx.Process(target=wrapped_func, args=[func, queue])
73
-
74
- process.start()
75
-
76
- process.join(timeout=ttl)
77
-
78
- if process.is_alive():
79
- process.terminate()
80
- process.join()
81
- raise TimeoutError(f"Failed to {func.func.__name__} after {ttl} seconds")
82
-
83
- # Raises an error if the queue is empty. This is fine. It means our subprocess timed out.
84
- result = queue.get(block=False)
85
-
86
- # If we put an exception on the queue then raise instead of returning.
87
- if isinstance(result, Exception):
88
- raise result
89
- if isinstance(result, BaseException):
90
- raise Exception(f"BaseException raised in subprocess: {str(result)}")
91
-
92
- return result
93
-
94
-
95
- def get_subtensor_and_metagraph() -> typing.Tuple[bt.subtensor, bt.metagraph]:
96
- for i in range(0, METAGRAPH_RETRIES):
97
- try:
98
- print("Connecting to subtensor...")
99
- subtensor: bt.subtensor = bt.subtensor(SUBTENSOR)
100
- print("Pulling metagraph...")
101
- metagraph: bt.metagraph = subtensor.metagraph(NETUID, lite=False)
102
- return subtensor, metagraph
103
- except:
104
- if i == METAGRAPH_RETRIES - 1:
105
- raise
106
- print(f"Error connecting to subtensor or pulling metagraph, retry {i + 1} of {METAGRAPH_RETRIES} in {METAGRAPH_DELAY_SECS} seconds...")
107
- time.sleep(METAGRAPH_DELAY_SECS)
108
- raise RuntimeError()
109
-
110
- @dataclass
111
- class ModelData:
112
- uid: int
113
- hotkey: str
114
- namespace: str
115
- name: str
116
- commit: str
117
- hash: str
118
- block: int
119
- incentive: float
120
- emission: float
121
- competition: str
122
-
123
- @classmethod
124
- def from_compressed_str(cls, uid: int, hotkey: str, cs: str, block: int, incentive: float, emission: float):
125
- """Returns an instance of this class from a compressed string representation"""
126
- tokens = cs.split(":")
127
- return ModelData(
128
- uid=uid,
129
- hotkey=hotkey,
130
- namespace=tokens[0],
131
- name=tokens[1],
132
- commit=tokens[2] if tokens[2] != "None" else "",
133
- hash=tokens[3] if tokens[3] != "None" else "",
134
- competition=tokens[4] if len(tokens) > 4 and tokens[4] != "None" else DEFAULT_COMPETITION_ID,
135
- block=block,
136
- incentive=incentive,
137
- emission=emission
138
- )
139
-
140
- def get_tao_price() -> float:
141
- for i in range(0, METAGRAPH_RETRIES):
142
- try:
143
- return float(requests.get("https://api.kucoin.com/api/v1/market/stats?symbol=TAO-USDT").json()["data"]["last"])
144
- except:
145
- if i == METAGRAPH_RETRIES - 1:
146
- raise
147
- time.sleep(METAGRAPH_DELAY_SECS)
148
- raise RuntimeError()
149
-
150
- def get_validator_weights(metagraph: bt.metagraph) -> typing.Dict[int, typing.Tuple[float, int, typing.Dict[int, float]]]:
151
- ret = {}
152
- for uid in metagraph.uids.tolist():
153
- vtrust = metagraph.validator_trust[uid].item()
154
- if vtrust > 0:
155
- ret[uid] = (vtrust, metagraph.S[uid].item(), {})
156
- for ouid in metagraph.uids.tolist():
157
- if ouid == uid:
158
- continue
159
- weight = round(metagraph.weights[uid][ouid].item(), 4)
160
- if weight > 0:
161
- ret[uid][-1][ouid] = weight
162
- return ret
163
-
164
- def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
165
- global last_refresh
166
- # Function to be executed in a thread
167
- def fetch_data(uid):
168
- hotkey = metagraph.hotkeys[uid]
169
- try:
170
- partial = functools.partial(get_metadata, subtensor, metagraph.netuid, hotkey)
171
- metadata = run_in_subprocess(partial, METADATA_TTL)
172
- except Exception as e:
173
- return None
174
-
175
- if not metadata:
176
- return None
177
-
178
- commitment = metadata["info"]["fields"][0]
179
- hex_data = commitment[list(commitment.keys())[0]][2:]
180
- chain_str = bytes.fromhex(hex_data).decode()
181
- block = metadata["block"]
182
- incentive = metagraph.incentive[uid].nan_to_num().item()
183
- emission = metagraph.emission[uid].nan_to_num().item() * 20 # convert to daily TAO
184
-
185
- try:
186
- model_data = ModelData.from_compressed_str(uid, hotkey, chain_str, block, incentive, emission)
187
- except Exception as e:
188
- return None
189
- return model_data
190
-
191
- # Use ThreadPoolExecutor to fetch data in parallel
192
- results = []
193
- with concurrent.futures.ThreadPoolExecutor() as executor:
194
- # Prepare the list of futures
195
- futures = [executor.submit(fetch_data, uid) for uid in metagraph.uids.tolist()]
196
- for future in tqdm(concurrent.futures.as_completed(futures), desc="Metadata for hotkeys", total=len(futures)):
197
- result = future.result()
198
- if result:
199
- results.append(result)
200
-
201
-
202
- last_refresh = datetime.datetime.now()
203
- return results
204
-
205
- def floatable(x) -> bool:
206
- return (isinstance(x, float) and not math.isnan(x) and not math.isinf(x)) or isinstance(x, int)
207
-
208
- def get_float_score(key: str, history, competition_id: str) -> typing.Tuple[typing.Optional[float], bool]:
209
- if key in history and "competition_id" in history:
210
- data = list(history[key])
211
- if len(data) > 0:
212
- competitions = list(history["competition_id"])
213
- while True:
214
- if competitions.pop() != competition_id:
215
- data.pop()
216
- continue
217
- if floatable(data[-1]):
218
- return float(data[-1]), True
219
- else:
220
- data = [float(x) for x in data if floatable(x)]
221
- if len(data) > 0:
222
- return float(data[-1]), False
223
- break
224
- return None, False
225
-
226
- def get_sample(uid, history, competition_id: str) -> typing.Optional[typing.Tuple[str, str, str]]:
227
- prompt_key = f"sample_prompt_data.{uid}"
228
- response_key = f"sample_response_data.{uid}"
229
- truth_key = f"sample_truth_data.{uid}"
230
- if prompt_key in history and response_key in history and truth_key in history and "competition_id" in history:
231
- competitions = list(history["competition_id"])
232
- prompts = list(history[prompt_key])
233
- responses = list(history[response_key])
234
- truths = list(history[truth_key])
235
- while True:
236
- prompt = prompts.pop()
237
- response = responses.pop()
238
- truth = truths.pop()
239
- if competitions.pop() != competition_id:
240
- continue
241
- if isinstance(prompt, str) and isinstance(response, str) and isinstance(truth, str):
242
- return prompt, response, truth
243
- break
244
- return None
245
-
246
- def get_scores(uids: typing.List[int], competition_id: str) -> typing.Dict[int, typing.Dict[str, typing.Optional[float | str]]]:
247
- api = wandb.Api()
248
- runs = list(api.runs(VALIDATOR_WANDB_PROJECT))
249
-
250
- result = {}
251
- for run in runs:
252
- history = run.history()
253
- for uid in uids:
254
- if uid in result.keys():
255
- continue
256
- perplexity, perplexity_fresh = get_float_score(f"perplexity_data.{uid}", history, competition_id)
257
- win_rate, win_rate_fresh = get_float_score(f"win_rate_data.{uid}", history, competition_id)
258
- win_total, win_total_fresh = get_float_score(f"win_total_data.{uid}", history, competition_id)
259
- weight, weight_fresh = get_float_score(f"weight_data.{uid}", history, competition_id)
260
- sample = get_sample(uid, history, competition_id)
261
- result[uid] = {
262
- "perplexity": perplexity,
263
- "win_rate": win_rate,
264
- "win_total": win_total,
265
- "weight": weight,
266
- "sample": sample,
267
- "fresh": perplexity_fresh and win_rate_fresh and win_total_fresh
268
- }
269
- if len(result.keys()) == len(uids):
270
- break
271
- return result
272
-
273
- def format_score(uid, scores, key) -> typing.Optional[float]:
274
- if uid in scores:
275
- if key in scores[uid]:
276
- point = scores[uid][key]
277
- if floatable(point):
278
- return round(scores[uid][key], 4)
279
- return None
280
-
281
- def next_tempo(start_block, tempo, block):
282
- start_num = start_block + tempo
283
- intervals = (block - start_num) // tempo
284
- nearest_num = start_num + ((intervals + 1) * tempo)
285
- return nearest_num
286
-
287
- subtensor, metagraph = get_subtensor_and_metagraph()
288
-
289
- tao_price = get_tao_price()
290
-
291
- leaderboard_df = get_subnet_data(subtensor, metagraph)
292
- leaderboard_df.sort(key=lambda x: x.incentive, reverse=True)
293
-
294
- competition_scores = {
295
- y.id: get_scores([x.uid for x in leaderboard_df if x.competition == y.id], y.id)
296
- for y in COMPETITIONS
297
- }
298
-
299
- current_block = metagraph.block.item()
300
- next_update = next_tempo(
301
- SUBNET_START_BLOCK,
302
- subtensor.get_subnet_hyperparameters(NETUID).tempo,
303
- current_block
304
- )
305
- blocks_to_go = next_update - current_block
306
- current_time = datetime.datetime.now()
307
- next_update_time = current_time + datetime.timedelta(seconds=blocks_to_go * SECONDS_PER_BLOCK)
308
-
309
- validator_df = get_validator_weights(metagraph)
310
- weight_keys = set()
311
- for uid, stats in validator_df.items():
312
- weight_keys.update(stats[-1].keys())
313
-
314
- def get_next_update():
315
- now = datetime.datetime.now()
316
- delta = next_update_time - now
317
- return f"""<div align="center" style="font-size: larger;">Next reward update: <b>{blocks_to_go}</b> blocks (~{int(delta.total_seconds() // 60)} minutes)</div>"""
318
-
319
- def leaderboard_data(show_stale: bool, scores: typing.Dict[int, typing.Dict[str, typing.Optional[float | str]]], competition_id: str):
320
- value = [
321
- [
322
- f'[{c.namespace}/{c.name} ({c.commit[0:8]}, UID={c.uid})](https://huggingface.co/{c.namespace}/{c.name}/commit/{c.commit})',
323
- format_score(c.uid, scores, "win_rate"),
324
- format_score(c.uid, scores, "perplexity"),
325
- format_score(c.uid, scores, "weight"),
326
- c.uid,
327
- c.block
328
- ] for c in leaderboard_df if c.competition == competition_id and (scores[c.uid]["fresh"] or show_stale)
329
- ]
330
- return value
331
-
332
- demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
333
- with demo:
334
- gr.HTML(FONT)
335
- gr.HTML(TITLE)
336
- gr.HTML(IMAGE)
337
- gr.HTML(HEADER)
338
-
339
- gr.HTML(value=get_next_update())
340
-
341
- with gr.Tabs():
342
- for competition in COMPETITIONS:
343
- with gr.Tab(competition.name):
344
- scores = competition_scores[competition.id]
345
- print(scores)
346
-
347
- class_denominator = sum(leaderboard_df[i].incentive for i in range(0, 10) if leaderboard_df[i].incentive and leaderboard_df[i].competition == competition.id)
348
-
349
- class_values = {
350
- f"{leaderboard_df[i].namespace}/{leaderboard_df[i].name} ({leaderboard_df[i].commit[0:8]}, UID={leaderboard_df[i].uid}) · ${round(leaderboard_df[i].emission * tao_price, 2):,} (τ{round(leaderboard_df[i].emission, 2):,})": \
351
- leaderboard_df[i].incentive / class_denominator for i in range(0, 10) if leaderboard_df[i].incentive and leaderboard_df[i].competition == competition.id
352
- }
353
-
354
- gr.Label(
355
- value=class_values,
356
- num_top_classes=10,
357
- )
358
-
359
- with gr.Accordion("Evaluation Stats"):
360
- gr.HTML(EVALUATION_HEADER.replace("{date}", last_refresh.strftime("refreshed at %H:%M on %Y-%m-%d")))
361
- with gr.Tabs():
362
- for entry in leaderboard_df:
363
- if entry.competition == competition.id:
364
- sample = scores[entry.uid]["sample"]
365
- if sample is not None:
366
- name = f"{entry.namespace}/{entry.name} ({entry.commit[0:8]}, UID={entry.uid})"
367
- with gr.Tab(name):
368
- gr.Chatbot([(sample[0], sample[1])])
369
- # gr.Chatbot([(sample[0], f"*{name}*: {sample[1]}"), (None, f"*GPT-4*: {sample[2]}")])
370
-
371
- show_stale = gr.Checkbox(label="Show Stale", interactive=True)
372
- leaderboard_table = gr.components.Dataframe(
373
- value=leaderboard_data(show_stale.value, scores, competition.id),
374
- headers=["Name", "Win Rate", "Perplexity", "Weight", "UID", "Block"],
375
- datatype=["markdown", "number", "number", "number", "number", "number"],
376
- elem_id="leaderboard-table",
377
- interactive=False,
378
- visible=True,
379
- )
380
- gr.HTML(EVALUATION_DETAILS)
381
- show_stale.change(lambda x: leaderboard_data(x, scores, competition.id), [show_stale], leaderboard_table)
382
-
383
- with gr.Accordion("Validator Stats"):
384
- validator_table = gr.components.Dataframe(
385
- value=[
386
- [uid, int(validator_df[uid][1]), round(validator_df[uid][0], 4)] + [validator_df[uid][-1].get(c.uid) for c in leaderboard_df if c.incentive]
387
- for uid, _ in sorted(
388
- zip(validator_df.keys(), [validator_df[x][1] for x in validator_df.keys()]),
389
- key=lambda x: x[1],
390
- reverse=True
391
- )
392
- ],
393
- headers=["UID", "Stake (τ)", "V-Trust"] + [f"{c.namespace}/{c.name} ({c.commit[0:8]}, UID={c.uid})" for c in leaderboard_df if c.incentive],
394
- datatype=["number", "number", "number"] + ["number" for c in leaderboard_df if c.incentive],
395
- interactive=False,
396
- visible=True,
397
- )
398
-
399
- def restart_space():
400
- API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
401
-
402
- scheduler = BackgroundScheduler()
403
- scheduler.add_job(restart_space, "interval", seconds=60*5) # restart every 15 minutes
404
- scheduler.start()
405
-
406
- demo.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
backup/nousgirl.png DELETED
Binary file (154 kB)
 
backup/requirements.txt DELETED
@@ -1,7 +0,0 @@
1
- bittensor==6.9.3
2
- requests==2.31.0
3
- wandb==0.16.2
4
- python-dotenv==1.0.1
5
- APScheduler==3.10.1
6
- huggingface-hub>=0.18.0
7
- tqdm==4.66.2
 
 
 
 
 
 
 
 
requirements.txt CHANGED
@@ -1,7 +1,7 @@
1
- bittensor==6.8.2
2
  requests==2.31.0
3
  wandb==0.16.2
4
  python-dotenv==1.0.1
5
  APScheduler==3.10.1
6
  huggingface-hub>=0.18.0
7
- tqdm==4.66.2
 
1
+ bittensor==6.9.3
2
  requests==2.31.0
3
  wandb==0.16.2
4
  python-dotenv==1.0.1
5
  APScheduler==3.10.1
6
  huggingface-hub>=0.18.0
7
+ tqdm==4.66.2