Update split_files_to_excel.py

split_files_to_excel.py

@@ -20,7 +20,9 @@ import unstructured
 from unstructured.partition.docx import partition_docx
 from unstructured.partition.auto import partition
 
-from transformers import AutoTokenizer
+
+import tiktoken
+#from transformers import AutoTokenizer
 
 from pypdf import PdfReader
 
@@ -40,14 +42,18 @@ embeddings = HuggingFaceEmbeddings(
 
 
 
-model_id = "mistralai/Mistral-7B-Instruct-v0.1"
-access_token = os.getenv("HUGGINGFACE_SPLITFILES_API_KEY")
+# model_id = "mistralai/Mistral-7B-Instruct-v0.1"
+# access_token = os.getenv("HUGGINGFACE_SPLITFILES_API_KEY")
+
+# tokenizer = AutoTokenizer.from_pretrained(
+#     model_id,
+#     padding_side="left",
+#     token = access_token
+# )
+
+
+tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo")
 
-tokenizer = AutoTokenizer.from_pretrained(
-    model_id,
-    padding_side="left",
-    token = access_token
-)
 
 text_splitter = CharacterTextSplitter(
     separator = "\n",