Fix: add repo_type=dataset and force_download=True for public dataset

chatbot_retriever.py

@@ -72,7 +72,10 @@ import os
 DATASET_REPO = "07Codex07/PrepGraph-Data"
 
 def ensure_data_dir():
-    """Ensure data/ folder exists and download PDFs from the Hugging Face dataset with correct subfolders."""
+    """Ensure data/ folder exists and contains the Hugging Face dataset PDFs properly structured."""
+    from huggingface_hub import hf_hub_download
+    import shutil
+
     data_dir = os.getenv("DATA_DIR", "data")
     os.makedirs(data_dir, exist_ok=True)
 
@@ -98,21 +101,25 @@ def ensure_data_dir():
 
     local_paths = []
     for f in files:
-        # ✅ Keep the original folder structure (e.g., data/pyqs/...)
-        download_path = os.path.join(data_dir, f)
-        os.makedirs(os.path.dirname(download_path), exist_ok=True)
+        dest_path = os.path.join(data_dir, f)          # ✅ keep real folder structure
+        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
 
-        if not os.path.exists(download_path):
+        if not os.path.exists(dest_path):
             print(f"📥 Downloading {f} from Hugging Face (public dataset)...")
             downloaded = hf_hub_download(
                 repo_id=DATASET_REPO,
                 filename=f,
-                repo_type="dataset",        # ✅ specify dataset type
-                force_download=True         # ✅ force refresh cache if needed
+                repo_type="dataset",
+                force_download=True,
             )
-            os.replace(downloaded, download_path)
+            shutil.copy(downloaded, dest_path)          # ✅ copy instead of rename (works inside HF Spaces)
+
+        local_paths.append(dest_path)
 
-        local_paths.append(download_path)
+    # Debug info for verification
+    print(f"✅ Total files ensured: {len(local_paths)}")
+    for p in local_paths[:3]:
+        print(f" → {p}")
 
     return local_paths