working commit

code/main.py

@@ -60,7 +60,47 @@ class Chatbot:
         )
 
         chain = cl.user_session.get("chain")
-        memory = chain.memory if chain else []
+        print(list(chain.store.values()))
+        memory_list = cl.user_session.get(
+            "memory",
+            (
+                list(chain.store.values())[0].messages
+                if len(chain.store.values()) > 0
+                else []
+            ),
+        )
+        conversation_list = []
+        for message in memory_list:
+            # Convert to dictionary if possible
+            message_dict = message.to_dict() if hasattr(message, "to_dict") else message
+
+            # Check if the type attribute is present as a key or attribute
+            message_type = (
+                message_dict.get("type", None)
+                if isinstance(message_dict, dict)
+                else getattr(message, "type", None)
+            )
+
+            # Check if content is present as a key or attribute
+            message_content = (
+                message_dict.get("content", None)
+                if isinstance(message_dict, dict)
+                else getattr(message, "content", None)
+            )
+
+            if message_type in ["ai", "ai_message"]:
+                conversation_list.append(
+                    {"type": "ai_message", "content": message_content}
+                )
+            elif message_type in ["human", "user_message"]:
+                conversation_list.append(
+                    {"type": "user_message", "content": message_content}
+                )
+            else:
+                raise ValueError("Invalid message type")
+        print("\n\n\n")
+        print("history at setup_llm", conversation_list)
+        print("\n\n\n")
 
         old_config = copy.deepcopy(self.config)
         self.config["vectorstore"]["db_option"] = retriever_method
@@ -71,7 +111,7 @@ class Chatbot:
         self.llm_tutor.update_llm(
             old_config, self.config
         )  # update only attributes that are changed
-        self.chain = self.llm_tutor.qa_bot(memory=memory)
+        self.chain = self.llm_tutor.qa_bot(memory=conversation_list)
 
         tags = [chat_profile, self.config["vectorstore"]["db_option"]]
 
@@ -222,7 +262,7 @@ class Chatbot:
         rename_dict = {"Chatbot": "AI Tutor"}
         return rename_dict.get(orig_author, orig_author)
 
-    async def start(self, thread=None, memory=[]):
+    async def start(self):
         """
         Start the chatbot, initialize settings widgets,
         and display and load previous conversation if chat logging is enabled.
@@ -236,6 +276,8 @@ class Chatbot:
         }
         print(self.user)
 
+        memory = cl.user_session.get("memory", [])
+
         cl.user_session.set("user", self.user)
         self.llm_tutor = LLMTutor(self.config, user=self.user)
         self.chain = self.llm_tutor.qa_bot(memory=memory)
@@ -273,6 +315,18 @@ class Chatbot:
         """
 
         chain = cl.user_session.get("chain")
+
+        print("\n\n\n")
+        print(
+            "session history",
+            chain.get_session_history(
+                self.user["user_id"],
+                self.user["session_id"],
+                self.config["llm_params"]["memory_window"],
+            ),
+        )
+        print("\n\n\n")
+
         llm_settings = cl.user_session.get("llm_settings", {})
         view_sources = llm_settings.get("view_sources", False)
         stream = (llm_settings.get("stream_response", True)) or (
@@ -318,28 +372,47 @@ class Chatbot:
             res, answer, stream=stream, view_sources=view_sources
         )
 
-        await cl.Message(content=answer_with_sources, elements=source_elements).send()
+        await cl.Message(
+            content=answer_with_sources, elements=source_elements, author=LLM
+        ).send()
 
     async def on_chat_resume(self, thread: ThreadDict):
         steps = thread["steps"]
-        conversation_pairs = []
+        # conversation_pairs = []
+        conversation_list = []
 
         user_message = None
         k = self.config["llm_params"]["memory_window"]
         count = 0
 
-        for step in steps:
-            if step["type"] == "user_message":
-                user_message = step["output"]
-            elif step["type"] == "assistant_message" and user_message is not None:
-                assistant_message = step["output"]
-                conversation_pairs.append((user_message, assistant_message))
-                user_message = None
-                count += 1
-                if count >= k:
-                    break
-
-        await self.start(thread, memory=conversation_pairs)
+        print(steps)
+
+        for step in reversed(steps):
+            print(step["type"])
+            if step["name"] not in [SYSTEM]:
+                if step["type"] == "user_message":
+                    conversation_list.append(
+                        {"type": "user_message", "content": step["output"]}
+                    )
+                elif step["type"] == "assistant_message":
+                    if step["name"] == LLM:
+                        conversation_list.append(
+                            {"type": "ai_message", "content": step["output"]}
+                        )
+                else:
+                    raise ValueError("Invalid message type")
+            count += 1
+            if count >= 2 * k:  # 2 * k to account for both user and assistant messages
+                break
+
+        conversation_list = conversation_list[::-1]
+
+        print("\n\n\n")
+        print("history at on_chat_resume", conversation_list)
+        print(len(conversation_list))
+        print("\n\n\n")
+        cl.user_session.set("memory", conversation_list)
+        await self.start()
 
     @cl.oauth_callback
     def auth_callback(

code/modules/chat/helpers.py

@@ -6,6 +6,11 @@ def get_sources(res, answer, stream=True, view_sources=False):
     source_elements = []
     source_dict = {}  # Dictionary to store URL elements
 
+    print("\n\n\n")
+    print(res["context"])
+    print(len(res["context"]))
+    print("\n\n\n")
+
     for idx, source in enumerate(res["context"]):
         source_metadata = source.metadata
         url = source_metadata.get("source", "N/A")
@@ -20,6 +25,9 @@ def get_sources(res, answer, stream=True, view_sources=False):
         source_type = source_metadata.get("source_type", "N/A")
 
         url_name = f"{url}_{page}"
+        print("url")
+        print(url_name)
+        print("\n\n\n")
         if url_name not in source_dict:
             source_dict[url_name] = {
                 "text": source.page_content,

code/modules/chat/langchain/langchain_rag.py

@@ -211,7 +211,7 @@ class Langchain_RAG_V2(BaseRAG):
         res = self.rag_chain.stream(user_query, config)
         return res
 
-    def add_history_from_list(self, history_list):
+    def add_history_from_list(self, conversation_list):
         """
         Add messages from a list to the chat history.
 
@@ -220,8 +220,22 @@ class Langchain_RAG_V2(BaseRAG):
         """
         history = ChatMessageHistory()
 
-        for idx, message_pairs in enumerate(history_list):
-            history.add_user_message(message_pairs[0])
-            history.add_ai_message(message_pairs[1])
+        for idx, message in enumerate(conversation_list):
+            message_type = (
+                message.get("type", None)
+                if isinstance(message, dict)
+                else getattr(message, "type", None)
+            )
+
+            message_content = (
+                message.get("content", None)
+                if isinstance(message, dict)
+                else getattr(message, "content", None)
+            )
+
+            if message_type in ["human", "user_message"]:
+                history.add_user_message(message_content)
+            elif message_type in ["ai", "ai_message"]:
+                history.add_ai_message(message_content)
 
         return history

code/modules/chat/langchain/utils.py

@@ -203,6 +203,7 @@ class CustomRunnableWithHistory(RunnableWithMessageHistory):
         print("Hist: ", hist)
         print("\n\n\n")
         messages = (await hist.aget_messages()).copy()
+        print("messages: ", messages)
 
         if not self.history_messages_key:
             # return all messages

code/modules/config/config.yml

@@ -3,7 +3,7 @@ log_chunk_dir: '../storage/logs/chunks' # str
 device: 'cpu' # str [cuda, cpu]
 
 vectorstore:
-  load_from_HF: True # bool
+  load_from_HF: False # bool
   embedd_files: False # bool
   data_path: '../storage/data' # str
   url_file_path: '../storage/data/urls.txt' # str

code/modules/dataloader/data_loader.py

@@ -228,11 +228,11 @@ class ChunkProcessor:
 
             page_num = doc.metadata.get("page", 0)
             file_data[page_num] = doc.page_content
-            metadata = (
-                addl_metadata.get(file_path, {})
-                if metadata_source == "file"
-                else {"source": file_path, "page": page_num}
-            )
+
+            # Create a new dictionary for metadata in each iteration
+            metadata = addl_metadata.get(file_path, {}).copy()
+            metadata["page"] = page_num
+            metadata["source"] = file_path
             file_metadata[page_num] = metadata
 
             if self.config["vectorstore"]["db_option"] not in ["RAGatouille"]: