change to regex based split

app.py

@@ -1,6 +1,7 @@
 import logging
 from pathlib import Path
 import os
+import re
 import gradio as gr
 import nltk
 import torch
@@ -29,7 +30,8 @@ def truncate_word_count(text, max_words=512):
     -------
     dict, the text and whether it was truncated
     """
-    words = text.split()
+    # split on whitespace with regex
+    words = re.split(r"\s+", text)
     processed = {}
     if len(words) > max_words:
         processed["was_truncated"] = True