Spaces:
Runtime error
Runtime error
File size: 590 Bytes
1bf52db 1482fe1 fdcca24 5fade0e d379f83 4087b07 3424db2 24a726e 5e833ac 17499bc 06d3c94 9531255 9116abe c990a96 e7b5de5 629049e 7be31bf 71873be 59fd5bc c3adb5e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
import yaml
from datasets import load_dataset
import pandas as pd
def make_dataset(dataset='cnn_dailymail', split='train'):
"""make dataset for summarisation"""
dataset = load_dataset(dataset, '3.0.0', split=split)
df = pd.DataFrame()
df['article'] = dataset['article']
df['highlights'] = dataset['highlights']
df.to_csv('data/raw/{}.csv'.format(split))
if __name__ == '__main__':
with open("params.yml") as f:
params = yaml.safe_load(f)
make_dataset(dataset=params['data'], split='train')
make_dataset(dataset=params['data'], split='test')
|