Update run.py

run.py

@@ -2,7 +2,7 @@
 # Title:  Gradio Interface to LLM-chatbot with dynamic RAG-funcionality and ChromaDB
 # Author: Andreas Fischer
 # Date:   October 10th, 2024
-# Last update: October 14th, 2024
+# Last update: October 15th, 2024
 ##########################################################################################
 
 import os
@@ -203,18 +203,18 @@ def add_doc(path, session):
     print(len(x))
     if(len(x)==0):
       chunkSize=40000
-      for i in range(round(len(corpus)/chunkSize+0.5)): #0 is first batch, 3 is last (incomplete) batch given 133497 texts
+      for i in range(round(len(corpus)/chunkSize+0.5)): 
         print("embed batch "+str(i)+" of "+str(round(len(corpus)/chunkSize+0.5)))
         ids=list(range(i*chunkSize,(i*chunkSize+chunkSize)))
         batch=corpus[i*chunkSize:(i*chunkSize+chunkSize)]
         textIDs=[str(id) for id in ids[0:len(batch)]]
         ids=[str(id+len(x)+1) for id in ids[0:len(batch)]] # id refers to chromadb-unique ID      
         collection.add(documents=batch, ids=ids, 
-          metadatas=[{"date": str("2024-10-10")} for b in batch]) #"textID":textIDs, "id":ids, 
+          metadatas=[{"date": str("2024-10-10")} for b in batch]) 
         print("finished batch "+str(i)+" of "+str(round(len(corpus)/40000+0.5)))  
     now = datetime.now()
     gr.Info(f"Indexing complete!")
-    print(now-then) #zu viel GB für sentences (GPU), bzw. 0:00:10.375087 für chunks
+    print(now-then) 
   return(collection)
 
 #split_with_overlap("test me if you can",2,1)