Spaces:
Runtime error
Runtime error
from transformers import AutoTokenizer | |
from datasets import load_dataset | |
tokenizer = AutoTokenizer.from_pretrained('distilbert-base-cased') | |
train_data = load_dataset('wikitext', 'wikitext-2-raw-v1', split='train') | |
token = train_data[10]['text'] | |
tokenized_train_data = tokenizer(token) | |
tokenized_train_data | |