Upload 13 files

app.py

@@ -0,0 +1,231 @@
+import streamlit as st
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain.output_parsers import CommaSeparatedListOutputParser
+from utils import fetch_wordpress_data, extract_text, generate_embeddings
+import chromadb, yaml
+from langchain_chroma import Chroma
+from langchain_community.embeddings.sentence_transformer import (
+    SentenceTransformerEmbeddings,
+)
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnableParallel, RunnablePassthrough
+from langchain_core.output_parsers import StrOutputParser
+from dotenv import load_dotenv
+load_dotenv()
+try:
+    # Attempt to load configuration data from config.yaml file
+    with open("./config.yaml", 'r') as file:
+        config_data = yaml.safe_load(file)
+except Exception as e:
+    # Raise exception if config.yaml file is not found
+    raise Exception(f"Not able to find the file ./config.yaml")
+
+# Set Streamlit page layout to wide
+st.set_page_config(layout="wide")
+
+# Initialize Chroma database client
+client = chromadb.PersistentClient("./posts_db") 
+collection_name = config_data['collection_name']
+collection = client.get_collection(name=collection_name)
+
+# Initialize embedding function for sentence transformer
+embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+
+# Initialize Langchain Chroma retriever
+langchain_chroma = Chroma(
+        client=client,
+        collection_name=collection_name,
+        embedding_function=embedding_function,
+    ).as_retriever()
+
+# Initialize Chat Google Generative AI model
+model = ChatGoogleGenerativeAI(model="gemini-pro")
+
+def update_vector_database(collection, post_id, embeddings, text):
+    """
+    Update the vector database with post ID, embeddings, and text.
+
+    Args:
+    collection: The collection in the database.
+    post_id (str): The ID of the post.
+    embeddings (list): List of embeddings generated from the post text.
+    text (str): The text of the post.
+    """
+    collection.upsert(ids=[str(post_id)], documents=[text], embeddings=[embeddings])
+
+def fetch_existing_posts(collection):
+    """
+    Fetch existing posts from the database.
+
+    Args:
+    collection: The collection in the database.
+
+    Returns:
+    list: List of existing posts.
+    """
+    # Fetch existing posts from the database or any other storage
+    existing_posts = collection.get()
+    return existing_posts
+
+def update_embeddings_on_new_post(collection):
+    """
+    Update embeddings for new posts fetched from WordPress data.
+
+    Args:
+    collection: The collection in the database.
+    """
+
+    # Fetch existing posts from the database or any other storage
+    existing_posts = fetch_existing_posts(collection)  
+    new_posts = fetch_wordpress_data(config_data['site_url'])
+
+    # Compare old and new posts to find the difference
+    existing_post_ids = set(str(post_id) for post_id in existing_posts['ids'])
+    new_posts_to_update = [post for post in new_posts if str(post['id']) not in existing_post_ids]
+    
+    # Update embeddings for new posts
+    for post in new_posts_to_update:
+        # Extract text from post
+        text = extract_text(post)  
+        # Generate embeddings for the post text
+        embeddings = generate_embeddings(text)
+        # Update vector database with post ID and embeddings
+        update_vector_database(collection,post['id'], embeddings, text)
+
+def rag_generate_response():
+    """
+    Generate a prompt for generating reasoning steps for answering the user query.
+
+    Returns:
+    ChatPromptTemplate: The generated prompt template.
+    """
+    template = """You are tasked with designing a prompt for generating reasoning steps for answering to the user_query in a website. Write a Prompt to generate a series of intermediate thoughts or reasoning steps to answer the query. Avoid providing specific solutions or examples, allowing the LLM to explore different approaches independently. Give the output as a list of steps. eg: [1,2,3,...]
+
+    Question: {user_query}
+    Previous_context : {previous_context}
+    """
+    prompt = ChatPromptTemplate.from_template(template)
+    return prompt
+    
+def develop_reasoning_steps(user_query, initial_prompt, previous_context):
+    """
+    Develop reasoning steps based on the user query, initial prompt, and previous context.
+
+    Args:
+    user_query (str): The query from the user.
+    initial_prompt (ChatPromptTemplate): The initial prompt template.
+    previous_context (str): The previous context.
+
+    Returns:
+    list: List of thought steps.
+    """
+    chain = (
+        RunnableParallel({"user_query": RunnablePassthrough(), "previous_context": RunnablePassthrough()})
+        | initial_prompt
+        | model
+        | CommaSeparatedListOutputParser()
+    )
+    thought_steps = chain.invoke({"user_query" : user_query, "previous_context" : previous_context})
+    thought_steps = thought_steps[0].split('\n')
+    return thought_steps
+
+def refine_response_based_on_thought_steps(user_query, thought_steps):
+    """
+    Refine the response based on thought steps.
+
+    Args:
+    user_query (str): The query from the user.
+    thought_steps (list): List of thought steps.
+
+    Returns:
+    str: Final refined response.
+    """
+    all_retrieved_content = ""
+    
+    for thought_step in thought_steps:
+        # print(langchain_chroma.invoke(thought_step))
+        retrieved_content = langchain_chroma.invoke(thought_step)
+        for i in retrieved_content:
+            all_retrieved_content+=i.page_content
+        all_retrieved_content+="\n"
+    template = """You are a helpful assistant which answers the query from the context. If the context does not provide the answer simply reply I cannot answer this and give a suggestion to refer the website. DO NOT say that 'there is no information in the context' or 'the answer from the context is this.' phrases, instead give directly the solution or answer I cannot answer this and give a suggestion to refer the website or similar kind of text based on the context.:
+    
+    query : {user_query}
+
+    context : {context}
+    """
+    prompt = ChatPromptTemplate.from_template(template)
+    reason_chain = (
+        RunnableParallel({'user_query': RunnablePassthrough(), 'context': RunnablePassthrough()})
+        | prompt
+        | model
+        | StrOutputParser()
+    )
+    final_response = reason_chain.invoke({'user_query': user_query ,'context' : all_retrieved_content})
+    return final_response
+    
+def process_query_with_chain_of_thought(user_query, previous_context):
+    """
+    Process the user query using the RAG + COT approach.
+
+    Args:
+    user_query (str): The query from the user.
+    previous_context (list): List of previous chat contexts.
+
+    Returns:
+    tuple: A tuple containing thought steps and final refined response.
+    """
+    initial_response = rag_generate_response(user_query)  # initial response is the prompt
+    thought_steps = develop_reasoning_steps(user_query, initial_response, previous_context)
+    final_response = refine_response_based_on_thought_steps(user_query,thought_steps)
+    return thought_steps, final_response
+
+def bot():
+    """
+    Streamlit application to run the conversational AI bot.
+    """
+    def web_bot():
+        global persist_directory
+        if st.button("New Chat 🤖",key="Start New Chat"):
+            st.session_state.clear()
+            st.session_state.app = web_bot
+            st.rerun()        
+
+        # Initialize chat history
+        if "messages" not in st.session_state:
+            st.session_state.messages = []
+
+
+        # Display chat messages from history and rerun
+        for message in st.session_state.messages:
+            with st.chat_message(message["role"]):
+                st.markdown(message["content"])
+                if message["role"]=="assistant":
+                    for i in message["thought_steps"]:
+                        st.markdown("- " + i)
+            
+        # Respond to user input after receiving
+        if user_query:= st.chat_input("What's up?"):
+            # Display user messages in chat message container
+            with st.chat_message("User"):
+                st.markdown(user_query)
+            # Add user message to chat history
+            st.session_state.messages.append({"role": "user", "content": user_query})
+
+            thought_steps, final_response = process_query_with_chain_of_thought(user_query, st.session_state.messages)
+        
+
+            # Display assistant response in hcat message container
+            with st.chat_message("assistant"):
+                for i in thought_steps:
+                    st.markdown("- " + i)
+                st.markdown(final_response)
+
+            st.session_state.messages.append({"role" : "assistant", "content": final_response, "thought_steps": thought_steps})
+
+    if 'app' not in st.session_state:
+        st.session_state.app = web_bot
+
+    st.session_state.app()
+
+bot()

config.yaml

@@ -0,0 +1,3 @@
+site_url: https://learn.wordpress.org
+collection_name: post_collection
+embedding_model: all-MiniLM-L6-v2

ingest.py

@@ -0,0 +1,30 @@
+from utils import fetch_wordpress_data,create_vector_store_and_add_posts
+import yaml
+
+def main():
+    """
+    Main function to fetch WordPress data, create vector store, and add posts to it.
+
+    This function reads configuration data from a YAML file, fetches WordPress data using the specified site URL,
+    and creates a vector store in the database with the fetched posts.
+
+    Raises:
+    Exception: If the config.yaml file is not found or if there are any other errors during execution.
+    """
+    try:
+        # Attempt to load configuration data from config.yaml file
+        with open("./config.yaml", 'r') as file:
+            config_data = yaml.safe_load(file)
+            print(config_data) # Printing configuration data for debugging purposes
+    except Exception as e:
+        # Raise exception if config.yaml file is not found
+        raise Exception(f"Not able to find the file ./config.yaml")
+    
+    # Fetch WordPress data using the site URL specified in the configuration
+    wordpress_data = fetch_wordpress_data(config_data['site_url'])
+
+    # Create vector store in the database and add WordPress posts to it
+    client, collection = create_vector_store_and_add_posts(wordpress_data)
+
+if __name__ == "__main__":
+    main()

posts_db/234f5c2c-1705-4f1b-b309-3779d27238a3/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3c9fd302f000d7790aa403c2d0d8fec363fe46f30b07d53020b6e33b22435a9
+size 1676000

posts_db/234f5c2c-1705-4f1b-b309-3779d27238a3/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e87a1dc8bcae6f2c4bea6d5dd5005454d4dace8637dae29bff3c037ea771411e
+size 100

posts_db/234f5c2c-1705-4f1b-b309-3779d27238a3/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:589714b29b763e833a33a2e5da994d6d404335449926d0341a8df001fab545cc
+size 4000

posts_db/234f5c2c-1705-4f1b-b309-3779d27238a3/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+size 0

query.py

@@ -0,0 +1,236 @@
+# Importing modules and functions
+from utils import fetch_wordpress_data, extract_text, generate_embeddings
+import chromadb, yaml
+from langchain_chroma import Chroma
+from langchain_community.embeddings.sentence_transformer import (
+    SentenceTransformerEmbeddings,
+)
+
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnableParallel, RunnablePassthrough
+from langchain.output_parsers import CommaSeparatedListOutputParser
+from langchain_core.output_parsers import StrOutputParser
+from dotenv import load_dotenv
+import argparse
+
+# Load environment variables from .env file
+load_dotenv()
+
+
+# Parse command-line arguments
+parser = argparse.ArgumentParser(description='Information retrieval from blog archives')
+parser.add_argument('--query', '-q', default="What are WordPress tutorials", type=str, help='What do you want to know...?', required=False)
+parser.add_argument('--chats', '-c', default=[("Hello", "Hey, How may i help you?")], type=list, help='Chats of the user', required=False)
+
+args = parser.parse_args()
+
+try:
+    # Attempt to load configuration data from config.yaml file
+    with open("./config.yaml", 'r') as file:
+        config_data = yaml.safe_load(file)
+except Exception as e:
+    # Raise exception if config.yaml file is not found
+    raise Exception(f"Not able to find the file ./config.yaml")
+
+# Initialize Chroma database client
+client = chromadb.PersistentClient("./posts_db") 
+collection_name = config_data['collection_name']
+collection = client.get_collection(name=collection_name)
+
+# Initialize embedding function for sentence transformer
+embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+
+# Initialize Langchain Chroma retriever
+langchain_chroma = Chroma(
+        client=client,
+        collection_name=collection_name,
+        embedding_function=embedding_function,
+    ).as_retriever(n_results=1)
+
+# Initialize Chat Google Generative AI model
+model = ChatGoogleGenerativeAI(model="gemini-pro")
+
+def update_vector_database(collection, post_id, embeddings, text):
+    """
+    Update the vector database with post ID, embeddings, and text.
+
+    Args:
+    collection: The collection in the database.
+    post_id (str): The ID of the post.
+    embeddings (list): List of embeddings generated from the post text.
+    text (str): The text of the post.
+    """
+    collection.upsert(ids=[str(post_id)], documents=[text], embeddings=[embeddings])
+
+def fetch_existing_posts(collection):
+    """
+    Fetch existing posts from the database.
+
+    Args:
+    collection: The collection in the database.
+
+    Returns:
+    list: List of existing posts.
+    """
+    # Fetch existing posts from the database or any other storage
+    existing_posts = collection.get()
+    return existing_posts
+
+def update_embeddings_on_new_post(collection):
+    """
+    Update embeddings for new posts fetched from WordPress data.
+
+    Args:
+    collection: The collection in the database.
+    """
+
+    # Fetch existing posts from the database or any other storage
+    existing_posts = fetch_existing_posts(collection)  
+    new_posts = fetch_wordpress_data(config_data['site_url'])
+
+    # Compare old and new posts to find the difference
+    # new_post_ids = set(str(post['id']) for post in new_posts)
+    existing_post_ids = set(str(post_id) for post_id in existing_posts['ids'])
+    new_posts_to_update = [post for post in new_posts if str(post['id']) not in existing_post_ids]
+    print(new_posts_to_update)
+    # Update embeddings for new posts
+    for post in new_posts_to_update:
+        # Extract text from post
+        text = extract_text(post)  
+        # Generate embeddings for the post text
+        embeddings = generate_embeddings(text)
+        # Update vector database with post ID and embeddings
+        update_vector_database(collection,post['id'], embeddings, text)
+
+def rag_generate_response():
+    """
+    Generate a prompt for generating reasoning steps for answering the user query.
+
+    Returns:
+    ChatPromptTemplate: The generated prompt template.
+    """
+    template = """You are tasked with designing a prompt for generating reasoning steps for answering to the user_query in a website. Write a Prompt to generate a series of intermediate thoughts or reasoning steps to answer the query. Avoid providing specific solutions or examples, allowing the LLM to explore different approaches independently. Give the output as a list of steps. eg: [1,2,3,...]
+
+    Question: {user_query}
+    Previous_context : {previous_context}
+    """
+    prompt = ChatPromptTemplate.from_template(template)
+    # chain = (
+    #     RunnableParallel({"user_query": RunnablePassthrough()})
+    #     | prompt
+    #     | model
+    #     | CommaSeparatedListOutputParser()
+    # )
+    # initial_response = chain.invoke(user_query)
+    # print("Initial Response: ", initial_response)
+    # return initial_response
+    return prompt
+    
+def develop_reasoning_steps(user_query, initial_prompt, previous_context):
+    """
+    Develop reasoning steps based on the user query, initial prompt, and previous context.
+
+    Args:
+    user_query (str): The query from the user.
+    initial_prompt (ChatPromptTemplate): The initial prompt template.
+    previous_context (str): The previous context.
+
+    Returns:
+    list: List of thought steps.
+    """
+    chain = (
+        RunnableParallel({"user_query": RunnablePassthrough(), "previous_context": RunnablePassthrough()})
+        | initial_prompt
+        | model
+        | CommaSeparatedListOutputParser()
+    )
+    thought_steps = chain.invoke({"user_query" : user_query, "previous_context" : previous_context})
+    thought_steps = thought_steps[0].split('\n')
+    print("thought_steps: ", thought_steps)
+    return thought_steps
+
+    # template = """You are a helpfull assistant which generates the thought steps used for generating a response to a query based on the relevant documents and the previous context. Generate thought steps on how to answer to the user's previous context with the relevant documents. This helps the LLM to answer in the next step accurately. I am giving the initial_response as relevant documents and previous context is the chat history. Now write the thought steps using the relavant documents, and the previous context on how to answer to the query. Thought steps are the sub questions and their answers asked using the query, context and previous context. Thought steps should be in the format Question: , Response: :
+    
+
+    # Relevant Document : {reasoning_step}
+
+    # previous context: {previous_context}
+    # """
+    # prompt = ChatPromptTemplate.from_template(template)
+    # reason_chain = (
+    #     RunnableParallel({"user_query": RunnablePassthrough(),"relevant_doc": RunnablePassthrough(), "previous_context": RunnablePassthrough()})
+    #     | prompt
+    #     | model
+    #     | StrOutputParser()
+    # )
+    # thought_steps = reason_chain.invoke({'user_query' : user_query, 'relevant_doc' : relevant_doc,'previous_context' : previous_context})
+    # return thought_steps
+
+def refine_response_based_on_thought_steps(user_query, thought_steps):
+    """
+    Refine the response based on thought steps.
+
+    Args:
+    user_query (str): The query from the user.
+    thought_steps (list): List of thought steps.
+
+    Returns:
+    str: Final refined response.
+    """
+    retrieved_content = []
+    all_retrieved_content = ""
+    
+    for thought_step in thought_steps:
+        # print(langchain_chroma.invoke(thought_step))
+        retrieved_content = langchain_chroma.invoke(thought_step)
+        for i in retrieved_content:
+            all_retrieved_content+=i.page_content
+        all_retrieved_content+="\n"
+
+    # print("--------------------------------------------------")
+    # print("Retrieved: ", retrieved_content[0])
+    # print("Len Retrieved: ", len(retrieved_content))
+    # print("----------------------------")
+    # print("All Retrieved Content: ",all_retrieved_content)
+    # print("Len Retrieved: ", len(retrieved_content[0]))
+    template = """You are a helpful assistant which answers the query from the context. If the context does not provide the answer simply reply I cannot answer this and give a suggestion to refer the website. DO NOT say that 'there is no information in the context' or 'the answer from the context is this.' phrases, instead give directly the solution or answer I cannot answer this and give a suggestion to refer the website or similar kind of text based on the context.:
+    
+    query : {user_query}
+
+    context : {context}
+    """
+    prompt = ChatPromptTemplate.from_template(template)
+    reason_chain = (
+        RunnableParallel({'user_query': RunnablePassthrough(), 'context': RunnablePassthrough()})
+        | prompt
+        | model
+        | StrOutputParser()
+    )
+    final_response = reason_chain.invoke({'user_query': user_query ,'context' : all_retrieved_content})
+    return final_response
+    
+# Function to process the query with RAG + COT
+def process_query_with_chain_of_thought(user_query, previous_context):
+    """
+    Process the user query using the RAG + COT approach.
+
+    Args:
+    user_query (str): The query from the user.
+    previous_context (list): List of previous chat contexts.
+
+    Returns:
+    tuple: A tuple containing thought steps and final refined response.
+    """
+    initial_response = rag_generate_response(user_query)  # initial response is the prompt
+    thought_steps = develop_reasoning_steps(user_query, initial_response, previous_context)
+    final_response = refine_response_based_on_thought_steps(user_query,thought_steps)
+    return thought_steps, final_response
+
+# Updating the embeddings and post content of the website into the vector store
+update_embeddings_on_new_post(collection)
+
+# Precessing the query from the user using the RAG + COT approach
+thought_steps, final_response = process_query_with_chain_of_thought(args.query, args.chats)
+print("Thought_steps : ",thought_steps)
+print("Final_response : ",final_response)

requirements.txt

@@ -0,0 +1,10 @@
+chromadb==0.4.24
+sentence-transformers==2.2.2
+langchain==0.1.17
+langchain-chroma==0.1.0
+langchain-community==0.0.37
+langchain-core==0.1.52
+langchain-google-genai==1.0.3
+python-dotenv==1.0.1
+streamlit==1.34.0
+google-generativeai==0.5.3

tests.py

@@ -0,0 +1,75 @@
+import unittest
+import chromadb 
+import subprocess
+import time
+import yaml
+import psutil
+from langchain_community.embeddings.sentence_transformer import (
+    SentenceTransformerEmbeddings,
+)
+from langchain_chroma import Chroma
+from langchain_google_genai import ChatGoogleGenerativeAI
+from dotenv import load_dotenv
+# Load environment variables from .env file
+load_dotenv()
+
+
+class TestChromaVectorStore(unittest.TestCase):
+    def setUp(self):
+        try:
+            # Attempt to load configuration data from config.yaml file
+            with open("./config.yaml", 'r') as file:
+                config_data = yaml.safe_load(file)
+        except Exception as e:
+            # Raise exception if config.yaml file is not found
+            raise Exception(f"Not able to find the file ./config.yaml")
+
+        self.client = chromadb.PersistentClient("./posts_db") 
+        collection_name = config_data['collection_name']
+        
+        self.collection = self.client.get_collection(name=collection_name)
+        # Initialize embedding function for sentence transformer
+        embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+
+        self.langchain_chroma = Chroma(
+                client=self.client,
+                collection_name=collection_name,
+                embedding_function=embedding_function,
+            ).as_retriever(n_results=1)
+    
+    def test_retrieve_vector_store(self):
+        # Testing whether the Chroma vector store retrieves data
+        data = self.langchain_chroma.invoke("Wordpress")
+        self.assertIsNotNone(data)
+        print("Vector Store is Working Properly!")
+        
+class TestLLM(unittest.TestCase):
+    def setUp(self):
+        self.model = ChatGoogleGenerativeAI(model="gemini-pro")
+    
+    def test_response_from_model(self):
+        # Testing whether LLM returns responses
+        response = self.model.invoke("Hello!")
+        self.assertIsNotNone(response)
+        print("LLM is generating responses!")
+
+class TestStreamlitUI(unittest.TestCase):
+    def test_streamlit_ui(self):
+        process = subprocess.Popen(["streamlit", "run", "app.py"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        time.sleep(60)
+        if process.poll() is None:
+            print("Streamlit app is running. Stopping the app...")
+
+            process_id = process.pid
+            parent = psutil.Process(process_id)
+            for child in parent.children(recursive=True):
+                child.terminate()
+            parent.terminate()
+            process.wait()
+            print("Streamlit is working properly.")
+        else:
+            print("Streamlit app has already terminated.")
+
+if __name__ == '__main__':
+    unittest.main()

utils.py

@@ -0,0 +1,122 @@
+import requests
+import re
+from html import unescape
+from sentence_transformers import SentenceTransformer
+import chromadb
+import yaml
+
+
+try:
+    # Attempt to load configuration data from config.yaml file
+    with open("./config.yaml", 'r') as file:
+        config_data = yaml.safe_load(file)
+except Exception as e:
+    # Raise exception if config.yaml file is not found
+    raise Exception(f"Not able to find the file ./config.yaml")
+
+
+# function to fetch data from WordPress site
+def fetch_wordpress_data(site_url):
+    """
+    Fetches data from a WordPress site using its REST API.
+
+    Args:
+    site_url (str): The URL of the WordPress site.
+
+    Returns:
+    dict: JSON data retrieved from the WordPress site.
+    """
+    api_url = f"{site_url}/wp-json/wp/v2/posts"
+    try:
+        # Send GET request to WordPress API
+        response = requests.get(api_url)
+        response.raise_for_status()  # Raise exception for unsuccessful responses
+
+        # Extract and return JSON data from response
+        return response.json()
+    
+    except requests.exceptions.RequestException as e:
+        # Handle any errors that occur during request
+        print("Error fetching WordPress data:", e)
+        return None
+
+def preprocess_text(text):
+    """
+    Preprocesses text by removing HTML tags, decoding special characters, and removing extra whitespaces.
+
+    Args:
+    text (str): The text to be preprocessed.
+
+    Returns:
+    str: The preprocessed text.
+    """
+    # Remove HTML tags
+    clean_text = re.sub('<.*?>', '', text)    
+    # Decode special characters
+    clean_text = unescape(clean_text)
+    # Removing extra newline characters
+    clean_text = re.sub('\n+', '\n', clean_text)    
+    # Remove extra whitespaces and newline characters
+    clean_text = clean_text.strip()
+    
+    return clean_text
+
+def generate_embeddings(text):
+    """
+    Generates sentence embeddings using a pre-trained embedding model.
+
+    Args:
+    text (str): The input text.
+
+    Returns:
+    list: List of sentence embeddings.
+    """
+    # Load pre-trained embedding model
+    model = SentenceTransformer(config_data['embedding_model'])
+
+    # Generate embeddings for input text
+    embeddings = model.encode(text)
+    return embeddings.tolist()
+
+def extract_text(post):
+    """
+    Extracts and preprocesses text content from a WordPress post.
+
+    Args:
+    post (dict): The WordPress post data.
+
+    Returns:
+    str: The preprocessed text content of the post.
+    """
+    return preprocess_text(post['content']['rendered'])
+
+def create_vector_store_and_add_posts(wordpress_data):
+    """
+    Creates a vector store in Chroma database and adds WordPress posts to it.
+
+    Args:
+    wordpress_data (list): List of WordPress post data.
+
+    Returns:
+    tuple: A tuple containing the Chroma client and collection objects.
+    """
+    client = chromadb.PersistentClient("./posts_db") 
+    collection = client.get_or_create_collection(name = config_data['collection_name'], metadata={"hnsw:space": "cosine"})
+    ids = []
+    documents = []
+    metadatas = []
+    embeddings = []
+    for post in wordpress_data:
+        ids.append(str(post['id']))
+        cleaned_content = extract_text(post)
+        embeddings.append(generate_embeddings(cleaned_content))
+        documents.append(cleaned_content)
+        metadata = {}
+        metadata['title'] = post['title']['rendered']
+        metadata['date'] = post['date']
+        metadata['modified'] = post['modified']
+        metadatas.append(metadata)
+    collection.upsert(ids=ids, documents=documents, metadatas=metadatas, embeddings=embeddings)
+    return client,collection
+
+