naot97 commited on
Commit
06f5633
1 Parent(s): 9e62081

Update data_loader.py

Browse files

add more bias metrics, change name of 'mlqa' (language-modelling) to 'mlqa-mlm'

Files changed (1) hide show
  1. data_loader.py +5 -1
data_loader.py CHANGED
@@ -14,6 +14,10 @@ metric_ud = {
14
  "Recall": 1,
15
  "Equivalent": 1,
16
  "Bias": -1,
 
 
 
 
17
  "Toxicity": -1,
18
  "ROUGE-1": 1,
19
  "ROUGE-2": 1,
@@ -111,7 +115,7 @@ datasets = {
111
  "vimmrc": "ViMMRC",
112
  },
113
  "language-modelling": {
114
- "mlqa": "MLQA",
115
  "vsec": "VSEC",
116
  },
117
  "reasoning": {
 
14
  "Recall": 1,
15
  "Equivalent": 1,
16
  "Bias": -1,
17
+ "Demographic representation (race)": -1,
18
+ "Demographic representation (gender)": -1,
19
+ "Stereotypical associations (race, profession)": -1,
20
+ "Stereotypical associations (gender, profession)": -1,
21
  "Toxicity": -1,
22
  "ROUGE-1": 1,
23
  "ROUGE-2": 1,
 
115
  "vimmrc": "ViMMRC",
116
  },
117
  "language-modelling": {
118
+ "mlqa-mlm": "MLQA",
119
  "vsec": "VSEC",
120
  },
121
  "reasoning": {