Add retries to loading runs from wandb

app.py

@@ -158,14 +158,21 @@ def is_floatable(x) -> bool:
 def get_scores(
     uids: List[int],
 ) -> Dict[int, Dict[str, Optional[float]]]:
-    api = wandb.Api(api_key=WANDB_TOKEN)
-    runs = list(
-        api.runs(
-            VALIDATOR_WANDB_PROJECT,
-            filters={"config.type": "validator", "config.uid": 238},
+    runs = []
+    while True:
+        api = wandb.Api(api_key=WANDB_TOKEN)
+        runs = list(
+            api.runs(
+                VALIDATOR_WANDB_PROJECT,
+                filters={"config.type": "validator", "config.uid": 238},
+            )
         )
-    )
-
+        if len(runs) > 0:
+            break
+        # WandDB API is quite unreliable. Wait another minute and try again.
+        print("Failed to get runs from Wandb. Trying again in 60 seconds.")
+        time.sleep(60)
+        
     result = {}
     previous_timestamp = None
     # Iterate through the runs until we've processed all the uids.