replaces nltk with re to find num of tokens

app.py

@@ -17,9 +17,9 @@ story_till_now = container_param.text_input(
     label='First Sentence', value='Hello, I\'m a language model,')
 
 num_generation = container_param.slider(
-    label='Number of generation', min_value=1, max_value=100, value=10, step=1)
+    label='Number of generation', min_value=1, max_value=100, value=3, step=1)
 length = container_param.slider(label='Length of the generated sentence',
-                                min_value=1, max_value=100, value=20, step=1)
+                                min_value=1, max_value=100, value=10, step=1)
 if mode == 'Create Statistics':
     container_mode.write('You selected statistics.')
     num_tests = container_param.slider(

story_gen.py

@@ -6,15 +6,15 @@ import printj
 from transformers import pipeline  # , set_seed
 import numpy as np
 import pandas as pd
-import nltk
-
+# import nltk
+import re
 
 class StoryGenerator:
     def __init__(self):
         self.initialise_models()
         self.stats_df = pd.DataFrame(data=[], columns=[])
         self.stories = []
-        nltk.download()
+        # nltk.download()
     
     
     def initialise_models(self):
@@ -39,7 +39,8 @@ class StoryGenerator:
 
     @staticmethod
     def get_num_token(text):
-        return len(nltk.word_tokenize(text))
+        # return len(nltk.word_tokenize(text))
+        return len(re.findall(r'\w+', text))
 
     @staticmethod
     def check_show_emotion(confidence_score, frequency, w):