Update app.py

app.py

@@ -44,6 +44,45 @@ load_dotenv()
 from langchain_community.document_loaders import TextLoader
 from langchain_experimental.text_splitter import SemanticChunker
 
+#####################
+from langchain import RecursiveCharacterTextSplitter
+from langchain_core.documents import BaseDocumentTransformer, Document
+
+class QQQSplitter(RecursiveCharacterTextSplitter):
+    def __init__(self):
+        super().__init__()
+
+    def split(self, documents: Iterable[Document]) -> List[Document]
+        """
+        Splits the given text whenever there is a "qqq" sequence.
+        """
+  
+        documents = []
+
+
+
+
+
+        for doc in documents: 
+            for char in doc.page_content:
+                if char == "q":
+                    if len(current_part) > 0 and current_part[-1] == "q":
+                        # Found a "qqq" sequence, split!
+                        parts.append(current_part[:-1])
+                        current_part = ""
+                    else:
+                        current_part += char
+                else:
+                    current_part += char
+    
+            parts.append(current_part)
+            print("cp " +current_part)
+            new_doc = Document(page_content=current_part, metadata=doc.metadata)
+            documents.append(new_doc)
+        
+        return documents
+##############################
+
 def load_txt(path="./a.cv.ckaller.2024.txt"):
     loader = TextLoader(path)
     document = loader.load()
@@ -60,13 +99,13 @@ def load_txt(path="./a.cv.ckaller.2024.txt"):
 
     ######
    # split the document into chunks
-    a_text_splitter = RecursiveCharacterTextSplitter( 
+     text_splitter = QQQSplitter( 
         chunk_size=1500,
         chunk_overlap=250,
         length_function=len,
         is_separator_regex=False,
     )
-    a_document_chunks = a_text_splitter.split_documents(document)
+     document_chunks =  text_splitter.split(document)
 
     
     ####### 
@@ -86,9 +125,9 @@ def load_txt(path="./a.cv.ckaller.2024.txt"):
 #####
 
 
-    text_splitter = SemanticChunker(HuggingFaceBgeEmbeddings())
+    #text_splitter = SemanticChunker(HuggingFaceBgeEmbeddings())
      
-    document_chunks = text_splitter.create_documents([state_of_the_union])
+    #document_chunks = text_splitter.create_documents([state_of_the_union])
     print(document_chunks[0].page_content)
 
     # load from disk