Update app.py

app.py

@@ -167,7 +167,7 @@ def select_embeddings_model(LLM_service="HuggingFace"):
     return embeddings
 
 #embeddings_OpenAI = select_embeddings_model(LLM_service="OpenAI")
-#embeddings_google = select_embeddings_model(LLM_service="Google")
+embeddings_google = select_embeddings_model(LLM_service="Google")
 embeddings_HuggingFace = select_embeddings_model(LLM_service="HuggingFace")
 
 
@@ -219,7 +219,7 @@ vector_store_OpenAI = Chroma(
 print("vector_store_OpenAI:",vector_store_OpenAI._collection.count(),"chunks.")
 """
 vector_store_google = Chroma(
-    persist_directory = LOCAL_VECTOR_STORE_DIR.as_posix() + "/Vit_All_Google_Embeddings",
+    persist_directory = current_dir + "/Vit_All_Google_Embeddings",
     embedding_function=embeddings_google)
 print("vector_store_google:",vector_store_google._collection.count(),"chunks.")
 
@@ -327,6 +327,117 @@ def CohereRerank_retriever(
 
 
 
+
+def retrieval_blocks(
+    create_vectorstore=True,# if True a Chroma vectorstore is created, else the Chroma vectorstore will be loaded
+    LLM_service="HuggingFace",
+    vectorstore_name="Vit_All_HF_Embeddings",
+    chunk_size = 1600, chunk_overlap=200, # parameters of the RecursiveCharacterTextSplitter
+    retriever_type="Vectorstore_backed_retriever",
+    base_retriever_search_type="similarity", base_retriever_k=10, base_retriever_score_threshold=None,
+    compression_retriever_k=16,
+    cohere_api_key="***", cohere_model="rerank-multilingual-v2.0", cohere_top_n=8,
+):
+    """
+    Rertieval includes: document loaders, text splitter, vectorstore and retriever. 
+    
+    Parameters: 
+        create_vectorstore (boolean): If True, a new Chroma vectorstore will be created. Otherwise, an existing vectorstore will be loaded.
+        LLM_service: OpenAI, Google or HuggingFace.
+        vectorstore_name (str): the name of the vectorstore.
+        chunk_size and chunk_overlap: parameters of the RecursiveCharacterTextSplitter, default = (1600,200).
+        
+        retriever_type (str): in [Vectorstore_backed_retriever,Contextual_compression,Cohere_reranker]
+        
+        base_retriever_search_type: search_type in ["similarity", "mmr", "similarity_score_threshold"], default = similarity.
+        base_retriever_k: The most similar vectors to retrieve (default k = 10).  
+        base_retriever_score_threshold: score_threshold used by the base retriever, default = None.
+
+        compression_retriever_k: top k documents returned by the compression retriever, default=16
+        
+        cohere_api_key: Cohere API key
+        cohere_model (str): The Cohere model can be either 'rerank-english-v2.0' or 'rerank-multilingual-v2.0', with the latter being the default.
+        cohere_top_n: top n results returned by Cohere rerank, default = 8.
+   
+    Output:
+        retriever.
+    """
+    try:
+        # Create new Vectorstore (Chroma index)
+        if create_vectorstore: 
+            # 1. load documents
+            documents = langchain_document_loader(current_dir)
+            
+            # 2. Text Splitter: split documents to chunks
+            text_splitter = RecursiveCharacterTextSplitter(
+                separators = ["\n\n", "\n", " ", ""],    
+                chunk_size = chunk_size,
+                chunk_overlap= chunk_overlap
+            )
+            chunks = text_splitter.split_documents(documents=documents)
+            
+            # 3. Embeddings
+            embeddings = select_embeddings_model(LLM_service=LLM_service)
+        
+            # 4. Vectorsore: create Chroma index
+            vector_store = create_vectorstore(
+                embeddings=embeddings,
+                documents = chunks,
+                vectorstore_name=vectorstore_name,
+            )
+    
+        # 5. Load a Vectorstore (Chroma index)
+        else: 
+            embeddings = select_embeddings_model(LLM_service=LLM_service)        
+            vector_store = Chroma(
+                persist_directory = current_dir + "/" + vectorstore_name,
+                embedding_function=embeddings
+            )
+            
+        # 6. base retriever: Vector store-backed retriever 
+        base_retriever = Vectorstore_backed_retriever(
+            vector_store,
+            search_type=base_retriever_search_type,
+            k=base_retriever_k,
+            score_threshold=base_retriever_score_threshold
+        )
+        retriever = None
+        if retriever_type=="Vectorstore_backed_retriever": 
+            retriever = base_retriever
+    
+        # 7. Contextual Compression Retriever
+        if retriever_type=="Contextual_compression":    
+            retriever = create_compression_retriever(
+                embeddings=embeddings,
+                base_retriever=base_retriever,
+                k=compression_retriever_k,
+            )
+    
+        # 8. CohereRerank retriever
+        if retriever_type=="Cohere_reranker":
+            retriever = CohereRerank_retriever(
+                base_retriever=base_retriever, 
+                cohere_api_key=cohere_api_key, 
+                cohere_model=cohere_model, 
+                top_n=cohere_top_n
+            )
+    
+        print(f"\n{retriever_type} is created successfully!")
+        print(f"Relevant documents will be retrieved from vectorstore ({vectorstore_name}) which uses {LLM_service} embeddings \
+and has {vector_store._collection.count()} chunks.")
+        
+        return retriever
+    except Exception as e:
+        print(e)
+
+
+
+
+
+
+
+
+
 def instantiate_LLM(LLM_provider,api_key,temperature=0.5,top_p=0.95,model_name=None):
     """Instantiate LLM in Langchain.
     Parameters: