File size: 1,429 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import json
import os
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET
from .base import BaseDataset
@LOAD_DATASET.register_module()
class commonsenseqaDataset(BaseDataset):
@staticmethod
def load(path):
dataset = {}
for split, stub in [
['train', 'train_rand_split.jsonl'],
['validation', 'dev_rand_split.jsonl'],
]:
data_path = os.path.join(path, stub)
dataset_list = []
with open(data_path, 'r', encoding='utf-8') as f:
for line in f:
line = json.loads(line)
dataset_list.append({
'question':
line['question']['stem'],
'A':
line['question']['choices'][0]['text'],
'B':
line['question']['choices'][1]['text'],
'C':
line['question']['choices'][2]['text'],
'D':
line['question']['choices'][3]['text'],
'E':
line['question']['choices'][4]['text'],
'answerKey':
line['answerKey'],
})
dataset[split] = Dataset.from_list(dataset_list)
return DatasetDict(dataset)
|