Clémentine commited on
Commit
bb17be3
1 Parent(s): ae85651

Fix TruthfulQA NaN scores to 0

Browse files
Files changed (1) hide show
  1. src/leaderboard/read_evals.py +2 -2
src/leaderboard/read_evals.py CHANGED
@@ -73,8 +73,8 @@ class EvalResult:
73
  continue
74
 
75
  # Some truthfulQA values are NaNs
76
- if task.benchmark == "truthfulqa:mc" and task.benchmark in data["results"]:
77
- if math.isnan(float(data["results"][task.benchmark][task.metric])):
78
  results[task.benchmark] = 0.0
79
  continue
80
 
 
73
  continue
74
 
75
  # Some truthfulQA values are NaNs
76
+ if task.benchmark == "truthfulqa:mc" and 'harness|truthfulqa:mc|0' in data["results"]:
77
+ if math.isnan(float(data["results"]['harness|truthfulqa:mc|0'][task.metric])):
78
  results[task.benchmark] = 0.0
79
  continue
80