Update core/rag_system.py

core/rag_system.py

@@ -70,68 +70,116 @@ class EnhancedRAGSystem:
         self.add_documents(english_data, english_metadatas)
     
     def add_documents(self, documents: List[str], metadatas: List[Dict] = None):
-        """Thêm documents vào database với embedding phù hợp"""
+        """Thêm documents vào database - ĐÃ SỬA LỖI"""
+        print(f"🔄 RAG System: Bắt đầu thêm {len(documents)} documents...")
+        
         if not documents:
+            print("❌ RAG System: Không có documents để thêm")
             return
         
         # Ensure metadatas has the same length as documents
         if metadatas is None:
             metadatas = [{} for _ in documents]
+            print("📝 Tạo metadata mặc định")
         elif len(metadatas) != len(documents):
-            # Extend or truncate metadatas to match documents length
-            if len(metadatas) < len(documents):
-                metadatas = metadatas + [{} for _ in range(len(documents) - len(metadatas))]
-            else:
-                metadatas = metadatas[:len(documents)]
+            print(f"⚠️ Metadata length mismatch: {len(metadatas)} vs {len(documents)}")
+            # Fix metadata length
+            new_metadatas = []
+            for i in range(len(documents)):
+                if i < len(metadatas):
+                    new_metadatas.append(metadatas[i])
+                else:
+                    new_metadatas.append({"source": "upload", "language": "vi"})
+            metadatas = new_metadatas
         
-        # Detect language for each document and create embeddings accordingly
-        new_embeddings_list = []
+        # Filter valid documents
         valid_documents = []
         valid_metadatas = []
         
         for i, doc in enumerate(documents):
-            if not doc or len(doc.strip()) == 0:
-                continue
+            if doc and isinstance(doc, str) and len(doc.strip()) > 5:  # At least 5 characters
+                valid_documents.append(doc.strip())
+                valid_metadatas.append(metadatas[i] if i < len(metadatas) else {})
+            else:
+                print(f"⚠️ Bỏ qua document {i}: không hợp lệ")
+        
+        print(f"📊 Documents hợp lệ: {len(valid_documents)}/{len(documents)}")
+        
+        if not valid_documents:
+            print("❌ Không có documents hợp lệ để thêm")
+            return
+        
+        # Create embeddings
+        new_embeddings_list = []
+        successful_embeddings = 0
+        
+        for i, doc in enumerate(valid_documents):
+            try:
+                language = valid_metadatas[i].get('language', 'vi')
+                embedding_model = self.multilingual_manager.get_embedding_model(language)
                 
-            language = metadatas[i].get('language', 'vi')
-            embedding_model = self.multilingual_manager.get_embedding_model(language)
-            
-            if embedding_model is not None:
-                try:
-                    # Create embedding for this document
-                    doc_embedding = embedding_model.encode([doc])
-                    new_embeddings_list.append(doc_embedding[0])
-                    valid_documents.append(doc)
-                    valid_metadatas.append(metadatas[i])
+                if embedding_model is None:
+                    print(f"⚠️ Không có embedding model cho document {i}")
+                    continue
                     
-                except Exception as e:
-                    print(f"❌ Lỗi tạo embedding cho document {i}: {e}")
+                # Create embedding
+                doc_embedding = embedding_model.encode([doc])
+                new_embeddings_list.append(doc_embedding[0])
+                successful_embeddings += 1
+                
+            except Exception as e:
+                print(f"❌ Lỗi embedding document {i}: {e}")
         
-        if not valid_documents:
+        print(f"📊 Embeddings thành công: {successful_embeddings}/{len(valid_documents)}")
+        
+        if not new_embeddings_list:
+            print("❌ Không tạo được embeddings nào")
             return
         
-        # Convert list of embeddings to numpy array
-        new_embeddings = np.array(new_embeddings_list)
+        # Convert to numpy array
+        try:
+            new_embeddings = np.array(new_embeddings_list)
+            print(f"✅ Embedding matrix shape: {new_embeddings.shape}")
+        except Exception as e:
+            print(f"❌ Lỗi tạo embedding matrix: {e}")
+            return
         
-        # Handle dimension mismatch
-        if self.embeddings is not None and self.embeddings.shape[1] != new_embeddings.shape[1]:
-            print(f"⚠️ Phát hiện dimension mismatch ({self.embeddings.shape[1]} vs {new_embeddings.shape[1]}), tạo index mới...")
-            self.embeddings = None
-            self.index = None
+        # Handle existing embeddings
+        old_doc_count = len(self.documents)
         
-        # Update embeddings
         if self.embeddings is None:
+            # First time initialization
             self.embeddings = new_embeddings
-            self.current_dimension = new_embeddings.shape[1]
+            self.documents = valid_documents
+            self.metadatas = valid_metadatas
+            print("✅ Khởi tạo RAG system lần đầu")
         else:
-            self.embeddings = np.vstack([self.embeddings, new_embeddings])
+            # Append to existing
+            try:
+                # Check dimension compatibility
+                if self.embeddings.shape[1] != new_embeddings.shape[1]:
+                    print(f"⚠️ Dimension mismatch: {self.embeddings.shape[1]} vs {new_embeddings.shape[1]}")
+                    print("🔄 Tạo system mới do dimension không khớp")
+                    self.embeddings = new_embeddings
+                    self.documents = valid_documents
+                    self.metadatas = valid_metadatas
+                else:
+                    # Compatible dimensions, append
+                    self.embeddings = np.vstack([self.embeddings, new_embeddings])
+                    self.documents.extend(valid_documents)
+                    self.metadatas.extend(valid_metadatas)
+                    print("✅ Đã thêm vào system hiện có")
+                    
+            except Exception as e:
+                print(f"❌ Lỗi khi thêm vào system: {e}")
+                return
         
         # Update FAISS index
         self._update_faiss_index()
         
-        self.documents.extend(valid_documents)
-        self.metadatas.extend(valid_metadatas)
-        print(f"✅ Đã thêm {len(valid_documents)} documents vào RAG database")
+        new_doc_count = len(self.documents)
+        print(f"🎉 THÀNH CÔNG: Đã thêm {new_doc_count - old_doc_count} documents mới")
+        print(f"📊 Tổng documents: {new_doc_count}")
     
     def _update_faiss_index(self):
         """Cập nhật FAISS index với embeddings hiện tại"""