ingest fix

src/ingest.py

@@ -1,50 +1,76 @@
 # src/ingest.py
 import pickle
-from io import BytesIO
 from pathlib import Path
 
 import faiss
+import pdfplumber
 from datasets import load_dataset
-from pypdf import PdfReader
 from sentence_transformers import SentenceTransformer
 
-from config import INDEX_DIR, FAISS_INDEX_PATH, DOCS_PATH, DATASET_NAME
+from config import (
+    DATASET_NAME,
+    FAISS_INDEX_PATH,
+    DOCS_PATH,
+    CHUNK_SIZE,
+    CHUNK_OVERLAP,
+)
 
 
-def build_index():
-    INDEX_DIR.mkdir(parents=True, exist_ok=True)
+def chunk_text(text: str):
+    chunks = []
+    start = 0
+
+    while start < len(text):
+        end = start + CHUNK_SIZE
+        chunks.append(text[start:end])
+        start = end - CHUNK_OVERLAP
+
+    return chunks
 
+
+def build_index():
+    print("📥 Loading HF dataset...")
     dataset = load_dataset(DATASET_NAME, split="train")
 
-    texts = []
+    embedder = SentenceTransformer("all-MiniLM-L6-v2")
 
-    for row in dataset:
-        pdf_obj = row["pdf"]   # HF auto column name
+    documents = []
+    embeddings = []
 
-        # HF PDF object → bytes
-        pdf_bytes = pdf_obj["bytes"]
-        reader = PdfReader(BytesIO(pdf_bytes))
+    print(f"📄 Dataset rows: {len(dataset)}")
 
-        for page in reader.pages:
-            text = page.extract_text()
-            if text:
-                texts.append(text.strip())
+    for row_idx, row in enumerate(dataset):
+        pdf_obj = row["pdf"]
 
-    if not texts:
+        # ✅ THIS IS THE KEY
+        pdf_path = pdf_obj.path
+
+        print(f"➡️ Processing PDF {row_idx + 1}: {pdf_path}")
+
+        with pdfplumber.open(pdf_path) as pdf:
+            for page in pdf.pages:
+                text = page.extract_text()
+                if not text:
+                    continue
+
+                for chunk in chunk_text(text):
+                    documents.append(chunk)
+                    embeddings.append(embedder.encode(chunk))
+
+    if not documents:
         raise RuntimeError("❌ No text extracted from PDFs")
 
-    # Embeddings
-    model = SentenceTransformer("all-MiniLM-L6-v2")
-    embeddings = model.encode(texts, show_progress_bar=True)
+    print(f"🧠 Creating FAISS index with {len(documents)} chunks")
 
-    # FAISS
-    dim = embeddings.shape[1]
+    dim = len(embeddings[0])
     index = faiss.IndexFlatL2(dim)
     index.add(embeddings)
 
+    FAISS_INDEX_PATH.parent.mkdir(parents=True, exist_ok=True)
+
     faiss.write_index(index, str(FAISS_INDEX_PATH))
 
     with open(DOCS_PATH, "wb") as f:
-        pickle.dump(texts, f)
+        pickle.dump(documents, f)
 
-    return index, texts
+    print("✅ FAISS index built successfully")