Update app.py

app.py

@@ -1,5 +1,5 @@
 from annotated_text import annotated_text, parameters, annotation
-from nltk.tokenize import word_tokenize
+import razdel
 from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
 import streamlit as st
 import torch
@@ -67,7 +67,7 @@ def label_text(text):
 				   49: 'B-TIME', 
 				   50: 'I-TIME'}
 
-		single_sentence_tokens = word_tokenize(text)
+		single_sentence_tokens = [_.text for _ in list(razdel.tokenize(text))]
 		tokenized_input = tokenizer(single_sentence_tokens, is_split_into_words = True, return_tensors = "pt")
 		tokens = tokenized_input.tokens()
 		output = model(**tokenized_input).logits