Fix: auto-download PDFs from HF dataset before loading

chatbot_retriever.py

@@ -66,6 +66,46 @@ SEARCH_EXPANSION = int(os.getenv("FAISS_SEARCH_EXPANSION", 5))
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+from huggingface_hub import hf_hub_download
+import os
+
+DATASET_REPO = "07Codex07/PrepGraph-Data"
+
+def ensure_data_dir():
+    """Make sure data/ folder exists and contains downloaded PDFs from HF dataset."""
+    data_dir = os.getenv("DATA_DIR", "data")
+    os.makedirs(data_dir, exist_ok=True)
+
+    files = [
+        "cn.pdf",
+        "dos.pdf",
+        "pyqs/cn_pyq_2019.pdf",
+        "pyqs/cn_pyq_2020.pdf",
+        "pyqs/cn_pyq_2022.pdf",
+        "pyqs/cn_pyq_2023.pdf",
+        "pyqs/cn_pyq_2024.pdf",
+        "pyqs/cn_pyq_2028.pdf",
+        "pyqs/dos_pyq_2019.pdf",
+        "pyqs/dos_pyq_2020.pdf",
+        "pyqs/dos_pyq_2024.pdf",
+        "pyqs/se_pyq_2018.pdf",
+        "pyqs/se_pyq_2019.pdf",
+        "pyqs/se_pyq_2020(S).pdf",
+        "pyqs/se_pyq_2020.pdf",
+        "pyqs/se_pyq_2022.pdf",
+        "pyqs/se_pyq_2024.pdf",
+    ]
+
+    local_paths = []
+    for f in files:
+        local_path = os.path.join(data_dir, f.replace("/", "_"))
+        if not os.path.exists(local_path):
+            print(f"📥 Downloading {f} from Hugging Face...")
+            downloaded = hf_hub_download(repo_id=DATASET_REPO, filename=f)
+            os.rename(downloaded, local_path)
+        local_paths.append(local_path)
+    return local_paths
+
 
 def detect_subject(fname: str) -> Optional[str]:
     # light heuristic to guess subject code from filename
@@ -151,7 +191,8 @@ def load_all_docs(base_dir: str = DATA_DIR) -> List:
 def build_or_load_indexes(force_reindex: bool = False):
     if os.getenv("FORCE_REINDEX", "0").lower() in ("1", "true", "yes"):
         force_reindex = True
-
+        
+    ensure_data_dir()
     docs = load_all_docs(DATA_DIR)
     if not docs:
         logger.warning("No documents found. Returning empty indexes.")