Add 200 token limit to pipeline

app.py

@@ -2,7 +2,7 @@ import streamlit as st
 import pandas as pd
 from preprocess_data import preprocess_text,get_stopwords
 from datasets import load_dataset
-from transformers import pipeline
+from transformers import AutoTokenizer, pipeline
 from wordnet import wordnet_pipeline
 
 dataset = load_dataset('danielcd99/imdb')
@@ -17,6 +17,9 @@ for split in dataset.keys():
 MODEL_PATH = 'danielcd99/BERT_imdb'
 
 def load_pipeline():
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
+    tokenizer.model_max_length = 200
+
     pipe=pipeline(
     "text-classification",
     model=MODEL_PATH