File size: 546 Bytes
1d4a13a 00a7216 1d4a13a 00a7216 1d4a13a 00a7216 1d4a13a 00a7216 1d4a13a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
from clean import clean_text
from datasets import load_dataset
dataset_v0 = load_dataset('oscar', "unshuffled_deduplicated_nl", split='train', streaming=True)
def f(obj):
obj["text"] = clean_text(obj["text"])
return obj
dataset_v1 = dataset_v0.map(f)
dataset_v2 = dataset_v1.filter(lambda obj: obj['text'] is not None)
it = iter(dataset_v0)
print(next(it))
print(next(it))
print(next(it))
it = iter(dataset_v1)
print(next(it))
print(next(it))
print(next(it))
it = iter(dataset_v2)
print(next(it))
print(next(it))
print(next(it))
|