Spaces:
Runtime error
Runtime error
Commit
•
5108918
1
Parent(s):
de1ce9e
Update app.py
Browse files
app.py
CHANGED
|
@@ -16,4 +16,7 @@ tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
|
|
| 16 |
def preprocess_function(examples):
|
| 17 |
return tokenizer(examples["text"], truncation=True)
|
| 18 |
|
| 19 |
-
tokenized_imdb = imdb.map(preprocess_function, batched=True)
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
def preprocess_function(examples):
|
| 17 |
return tokenizer(examples["text"], truncation=True)
|
| 18 |
|
| 19 |
+
tokenized_imdb = imdb.map(preprocess_function, batched=True)
|
| 20 |
+
|
| 21 |
+
from transformers import DataCollatorWithPadding
|
| 22 |
+
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|