Update utils.py

utils.py

@@ -348,11 +348,20 @@ def document_loading_splitting():
     # Document splitting
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)# RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
     splits = text_splitter.split_documents(preprocessed_docs)
+    # Split sowohl für originale als auch für vorverarbeitete Dokumente
+    original_splits = text_splitter.split_documents(docs)
+    preprocessed_splits = text_splitter.split_documents(preprocessed_docs)
+    
+    # Mapping von vorverarbeiteten Splits zu Originalsplits
+    split_to_original_mapping = {p_split: o_split for p_split, o_split in zip(preprocessed_splits, original_splits)}
+
+
     print("Splits...........................")
-    for split in splits:
+    for split in preprocessed_splits:
         if 'divis' in split.page_content:
             print("DIVIS found in chunk:", split)
-    return splits
+            
+    return preprocessed_splits, split_to_original_mapping
 
 ###########################################
 #Chroma DB die splits ablegen - vektorisiert...