MelikeDulkadir commited on
Commit
de1ce9e
1 Parent(s): 3f44762

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -1
app.py CHANGED
@@ -14,4 +14,6 @@ from transformers import AutoTokenizer
14
  tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
15
 
16
  def preprocess_function(examples):
17
- return tokenizer(examples["text"], truncation=True)
 
 
 
14
  tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
15
 
16
  def preprocess_function(examples):
17
+ return tokenizer(examples["text"], truncation=True)
18
+
19
+ tokenized_imdb = imdb.map(preprocess_function, batched=True)