Updated 5th chapter of OS

rag/ingest.py

@@ -1,5 +1,3 @@
-# rag/ingest.py
-
 import os
 import torch
 from dotenv import load_dotenv
@@ -11,25 +9,25 @@ from tqdm import tqdm
 
 load_dotenv()
 
-# ── Config ──────────────────────────────────────────────────────────────────
-PINECONE_API_KEY   = os.getenv("PINECONE_API_KEY")
-PINECONE_INDEX     = os.getenv("PINECONE_INDEX", "study-saathi")
-EMBEDDING_MODEL    = "intfloat/multilingual-e5-large"
-DATA_DIR           = "data/os_notes"
-CHUNK_SIZE         = 512
-CHUNK_OVERLAP      = 64
-BATCH_SIZE         = 32
-DIMENSION          = 1024   # multilingual-e5-large output dim
-
-# ── Device ──────────────────────────────────────────────────────────────────
+# ── Config ───────────────────────────────────────────────────────────────────
+PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
+PINECONE_INDEX   = os.getenv("PINECONE_INDEX", "study-saathi")
+EMBEDDING_MODEL  = "intfloat/multilingual-e5-large"
+DATA_DIR         = "data/os_notes"
+CHUNK_SIZE       = 512
+CHUNK_OVERLAP    = 64
+BATCH_SIZE       = 32
+DIMENSION        = 1024
+
+# ── Device ───────────────────────────────────────────────────────────────────
 device = "cuda" if torch.cuda.is_available() else "cpu"
 print(f"[INFO] Using device: {device}")
 
-# ── Load Embedding Model ─────────────────────────────────────────────────────
+# ── Load Embedding Model ──────────────────────────────────────────────────────
 print("[INFO] Loading embedding model...")
 embedder = SentenceTransformer(EMBEDDING_MODEL, device=device)
 
-# ── Pinecone Setup ───────────────────────────────────────────────────────────
+# ── Pinecone Setup ────────────────────────────────────────────────────────────
 pc = Pinecone(api_key=PINECONE_API_KEY)
 
 if PINECONE_INDEX not in [i.name for i in pc.list_indexes()]:
@@ -43,29 +41,46 @@ if PINECONE_INDEX not in [i.name for i in pc.list_indexes()]:
 
 index = pc.Index(PINECONE_INDEX)
 
-# ── Load Documents ───────────────────────────────────────────────────────────
-def load_documents(data_dir: str) -> list:
-    docs = []
-    for filename in os.listdir(data_dir):
-        filepath = os.path.join(data_dir, filename)
-        if filename.endswith(".pdf"):
-            loader = PyPDFLoader(filepath)
-        elif filename.endswith(".txt"):
-            loader = TextLoader(filepath, encoding="utf-8")
-        else:
-            print(f"[SKIP] Unsupported file: {filename}")
-            continue
-        loaded = loader.load()
-        # attach filename as topic metadata
-        topic = os.path.splitext(filename)[0]
-        for doc in loaded:
-            doc.metadata["topic"] = topic
-            doc.metadata["source"] = filename
-        docs.extend(loaded)
-        print(f"[LOADED] {filename} — {len(loaded)} page(s)")
-    return docs
-
-# ── Chunk Documents ──────────────────────────────────────────────────────────
+# ── Check if file already ingested ───────────────────────────────────────────
+def is_already_ingested(filename: str) -> bool:
+    """
+    Query Pinecone for any vector whose metadata source == filename.
+    If found, the file was already ingested — skip it.
+    """
+    topic = os.path.splitext(filename)[0]
+
+    # use a dummy zero vector just to run a metadata filter query
+    dummy_vector = [0.0] * DIMENSION
+
+    results = index.query(
+        vector=dummy_vector,
+        top_k=1,
+        include_metadata=True,
+        filter={"source": {"$eq": filename}}
+    )
+
+    return len(results["matches"]) > 0
+
+# ── Load Documents ────────────────────────────────────────────────────────────
+def load_documents(filepath: str, filename: str) -> list:
+    if filename.endswith(".pdf"):
+        loader = PyPDFLoader(filepath)
+    elif filename.endswith(".txt"):
+        loader = TextLoader(filepath, encoding="utf-8")
+    else:
+        return []
+
+    loaded = loader.load()
+    topic  = os.path.splitext(filename)[0]
+
+    for doc in loaded:
+        doc.metadata["topic"]  = topic
+        doc.metadata["source"] = filename
+
+    print(f"[LOADED] {filename} — {len(loaded)} page(s)")
+    return loaded
+
+# ── Chunk Documents ───────────────────────────────────────────────────────────
 def chunk_documents(docs: list) -> list:
     splitter = RecursiveCharacterTextSplitter(
         chunk_size=CHUNK_SIZE,
@@ -75,26 +90,23 @@ def chunk_documents(docs: list) -> list:
     print(f"[INFO] Total chunks: {len(chunks)}")
     return chunks
 
-# ── Embed & Upsert ───────────────────────────────────────────────────────────
-def embed_and_upsert(chunks: list):
-    texts = [
-        f"passage: {chunk.page_content}" for chunk in chunks  # e5 prefix
-    ]
+# ── Embed & Upsert ────────────────────────────────────────────────────────────
+def embed_and_upsert(chunks: list, filename: str):
+    texts = [f"passage: {chunk.page_content}" for chunk in chunks]
     print("[INFO] Generating embeddings...")
     all_vectors = []
 
     for i in tqdm(range(0, len(texts), BATCH_SIZE)):
-        batch_texts   = texts[i: i + BATCH_SIZE]
-        batch_chunks  = chunks[i: i + BATCH_SIZE]
-        embeddings    = embedder.encode(
+        batch_texts  = texts[i: i + BATCH_SIZE]
+        batch_chunks = chunks[i: i + BATCH_SIZE]
+        embeddings   = embedder.encode(
             batch_texts,
             normalize_embeddings=True,
             show_progress_bar=False
         )
-        vectors = []
         for j, (emb, chunk) in enumerate(zip(embeddings, batch_chunks)):
-            vectors.append({
-                "id":     f"chunk-{i + j}",
+            all_vectors.append({
+                "id":     f"{os.path.splitext(filename)[0]}-chunk-{i + j}",
                 "values": emb.tolist(),
                 "metadata": {
                     "text":   chunk.page_content,
@@ -102,20 +114,39 @@ def embed_and_upsert(chunks: list):
                     "source": chunk.metadata.get("source", "unknown"),
                 }
             })
-        all_vectors.extend(vectors)
 
-    # upsert in batches of 100 (Pinecone limit)
     print("[INFO] Upserting to Pinecone...")
     for i in tqdm(range(0, len(all_vectors), 100)):
         index.upsert(vectors=all_vectors[i: i + 100])
 
-    print(f"[DONE] Upserted {len(all_vectors)} chunks to Pinecone.")
+    print(f"[DONE] Upserted {len(all_vectors)} chunks for '{filename}'.")
 
-# ── Main ─────────────────────────────────────────────────────────────────────
+# ── Main ──────────────────────────────────────────────────────────────────────
 if __name__ == "__main__":
-    docs   = load_documents(DATA_DIR)
-    if not docs:
-        print("[ERROR] No documents found in data/os_notes/")
+    files = [f for f in os.listdir(DATA_DIR) if f.endswith((".pdf", ".txt"))]
+
+    if not files:
+        print("[ERROR] No files found in data/os_notes/")
         exit(1)
-    chunks = chunk_documents(docs)
-    embed_and_upsert(chunks)
+
+    print(f"[INFO] Found {len(files)} file(s): {files}\n")
+
+    for filename in files:
+        filepath = os.path.join(DATA_DIR, filename)
+
+        # ── SKIP CHECK ────────────────────────────────────────────────────
+        if is_already_ingested(filename):
+            print(f"[SKIP] '{filename}' already in Pinecone. Skipping...\n")
+            continue
+
+        print(f"[NEW]  Processing '{filename}'...")
+        docs   = load_documents(filepath, filename)
+        if not docs:
+            print(f"[WARN] Could not load '{filename}'. Skipping.\n")
+            continue
+
+        chunks = chunk_documents(docs)
+        embed_and_upsert(chunks, filename)
+        print()
+
+    print("[ALL DONE] Ingestion complete. Existing embeddings are untouched.")

static/js/design.js

@@ -25,7 +25,6 @@ const sidebarClose = document.getElementById("sidebarClose");
 const quickBtns    = document.querySelectorAll(".quick-btn");
 
 // ── Conversation history ──────────────────────────────────────────────────────
-// Each entry: { role: "user" | "assistant", content: "..." }
 let history = [];
 
 // ── Sidebar toggle ────────────────────────────────────────────────────────────
@@ -44,16 +43,18 @@ sidebarClose.addEventListener("click", closeSidebar);
 overlay.addEventListener("click", closeSidebar);
 
 // ── Auto-resize textarea ──────────────────────────────────────────────────────
-queryInput.addEventListener("input", () => {
+function autoResize() {
   queryInput.style.height = "auto";
-  queryInput.style.height = Math.min(queryInput.scrollHeight, 140) + "px";
-});
+  queryInput.style.height = Math.min(queryInput.scrollHeight, 200) + "px";
+}
+
+queryInput.addEventListener("input", autoResize);
 
 // ── Quick prompt buttons ──────────────────────────────────────────────────────
 quickBtns.forEach(btn => {
   btn.addEventListener("click", () => {
     queryInput.value = btn.dataset.prompt;
-    queryInput.dispatchEvent(new Event("input"));
+    autoResize();
     queryInput.focus();
     closeSidebar();
   });
@@ -61,7 +62,7 @@ quickBtns.forEach(btn => {
 
 // ── Clear chat ────────────────────────────────────────────────────────────────
 clearBtn.addEventListener("click", () => {
-  history = [];   // reset memory
+  history = [];
   const welcome = document.getElementById("welcomeMsg");
   chatWindow.innerHTML = "";
   if (welcome) chatWindow.appendChild(welcome);
@@ -88,6 +89,8 @@ function addMessage(role, content) {
     bubble.innerHTML = marked.parse(content);
     bubble.querySelectorAll("pre code").forEach(el => hljs.highlightElement(el));
   } else {
+    // user bubble: preserve newlines and show code syntax as plain text
+    bubble.style.whiteSpace = "pre-wrap";
     bubble.textContent = content;
   }
 
@@ -132,10 +135,7 @@ async function sendMessage() {
 
   const topic = topicInput.value.trim() || null;
 
-  // show user bubble
   addMessage("user", query);
-
-  // push to history BEFORE sending
   history.push({ role: "user", content: query });
 
   // reset input
@@ -159,7 +159,6 @@ async function sendMessage() {
       addMessage("assistant", `⚠️ **Error:** ${data.error}`);
     } else {
       addMessage("assistant", data.response);
-      // push assistant reply into history
       history.push({ role: "assistant", content: data.response });
     }
 
@@ -172,13 +171,20 @@ async function sendMessage() {
   }
 }
 
-// ── Send on button click ──────────────────────────────────────────────────────
-sendBtn.addEventListener("click", sendMessage);
-
-// ── Send on Enter (Shift+Enter = new line) ────────────────────────────────────
+// ── Keyboard handling ─────────────────────────────────────────────────────────
 queryInput.addEventListener("keydown", (e) => {
-  if (e.key === "Enter" && !e.shiftKey) {
+  if (e.key === "Enter") {
+    if (e.shiftKey) {
+      // Shift+Enter — insert a real newline and resize
+      // let the browser insert \n naturally, then resize
+      setTimeout(autoResize, 0);
+      return;                       // do NOT send
+    }
+    // plain Enter — send
     e.preventDefault();
     sendMessage();
   }
-});
+});
+
+// ── Send button ───────────────────────────────────────────────────────────────
+sendBtn.addEventListener("click", sendMessage);