Update app.py

app.py

@@ -158,25 +158,29 @@ class GraphState(TypedDict):
     decision: str
     documents: List[str]
 
+import os
+from bs4 import BeautifulSoup
+import pandas as pd
+
 def process_documents(temp_dir):
     """Process documents from the extracted zip folder with enhanced error handling."""
     d = {"chunk": [], "url": []}
     
     # Debug information
     print(f"Scanning directory: {temp_dir}")
-    print(f"Directory contents: {os.listdir(temp_dir)}")
     
     file_count = 0
     processed_count = 0
     error_count = 0
     
-    for path in os.listdir(temp_dir):
-        file_count += 1
-        if os.path.isfile(os.path.join(temp_dir, path)):
+    # Recursively traverse the directory
+    for root, dirs, files in os.walk(temp_dir):
+        for file_name in files:
+            file_count += 1
+            file_path = os.path.join(root, file_name)
+            print(f"Processing file: {file_path}")
+            
             try:
-                file_path = os.path.join(temp_dir, path)
-                print(f"Processing file: {path}")
-                
                 # Try different encodings
                 encodings = ['utf-8', 'latin-1', 'cp1252']
                 content = None
@@ -190,7 +194,7 @@ def process_documents(temp_dir):
                         continue
                 
                 if content is None:
-                    print(f"Failed to read file {path} with any encoding")
+                    print(f"Failed to read file {file_path} with any encoding")
                     error_count += 1
                     continue
                 
@@ -203,19 +207,19 @@ def process_documents(temp_dir):
                 text_content = main_content.get_text(strip=True) if main_content else soup.get_text(strip=True)
                 
                 if not text_content.strip():
-                    print(f"No content extracted from {path}")
+                    print(f"No content extracted from {file_path}")
                     error_count += 1
                     continue
                 
                 full_content = f"{title_text}\n\n{text_content}"
                 
                 d["chunk"].append(full_content)
-                d["url"].append("https://" + path.replace("=", "/"))
+                d["url"].append("https://" + file_name.replace("=", "/"))
                 processed_count += 1
-                print(f"Successfully processed {path}")
+                print(f"Successfully processed {file_path}")
                 
             except Exception as e:
-                print(f"Error processing file {path}: {str(e)}")
+                print(f"Error processing file {file_path}: {str(e)}")
                 error_count += 1
                 continue
     
@@ -348,6 +352,11 @@ def handle_upload(zip_file, csv_file):
                 zip_ref.extractall(temp_dir)
                 print(f"ZIP contents: {zip_ref.namelist()}")
             
+            # Process documents
+            print("Processing documents...")
+            df = process_documents(temp_dir)
+            print(f"Processed {len(df)} documents")
+            
             # Preprocess and read requirements CSV
             print("Processing CSV file...")
             requirements_df = preprocess_csv(csv_file)