Revert to nltk word tokenizer

Summarizer.py

@@ -1,4 +1,5 @@
 import string
+import nltk
 
 from sumy.parsers import DocumentParser
 from sumy.parsers.html import HtmlParser
@@ -7,13 +8,12 @@ from sumy.nlp.tokenizers import Tokenizer
 from sumy.nlp.stemmers import Stemmer
 from sumy.summarizers.lsa import LsaSummarizer
 from sumy.utils import get_stop_words
-from transformers import Pipeline, BertTokenizer
+from transformers import Pipeline
 
 
 class Summarizer:
     DEFAULT_LANGUAGE = "english"
     DEFAULT_EXTRACTED_ARTICLE_SENTENCES_LENGTH = 10
-    TOKENIZER = BertTokenizer.from_pretrained('bert-base-cased')
     STOP_WORDS = list(get_stop_words(language=DEFAULT_LANGUAGE)) + list(string.punctuation)
 
     def __init__(self, pipeline: Pipeline):
@@ -40,7 +40,7 @@ class Summarizer:
         cumulative_token_length = 0
         for sentence in summary_sentences:
             result_list.append(sentence)
-            token_list = Summarizer.TOKENIZER.tokenize(sentence)
+            token_list = nltk.word_tokenize(sentence)
             token_words = [token for token in token_list if token.lower() not in Summarizer.STOP_WORDS]
             token_length = len(token_words)
             if token_length + cumulative_token_length >= max_token_length: