working on text splitting

app.py

@@ -58,13 +58,14 @@ def main() -> None:
             # token_list = [token for token in nltk.word_tokenize(sentence)]
             token_list = tokenizer(sentence, max_length=1024, truncation=True)
             token_length = len(token_list["input_ids"])
-            if token_length + cumulative_token_length > split_token_length and result_list:
-                accumulated_lists.append(join_sentences(result_list))
-                result_list = [sentence]
-                cumulative_token_length = token_length
-            else:
-                result_list.append(sentence)
-                cumulative_token_length += token_length
+            if token_length > 10:
+                if token_length + cumulative_token_length > split_token_length and result_list:
+                    accumulated_lists.append(join_sentences(result_list))
+                    result_list = [sentence]
+                    cumulative_token_length = token_length
+                else:
+                    result_list.append(sentence)
+                    cumulative_token_length += token_length
         if result_list:
             accumulated_lists.append(join_sentences(result_list))
         return accumulated_lists