Upload 8 files

.gitattributes

@@ -51,3 +51,6 @@ Dataset/Penjas/PJOK_BS_KLS_III.pdf filter=lfs diff=lfs merge=lfs -text
 Dataset/Penjas/PJOK_BS_KLS_IV.pdf filter=lfs diff=lfs merge=lfs -text
 Dataset/Penjas/PJOK_BS_KLS_V.pdf filter=lfs diff=lfs merge=lfs -text
 Dataset/Penjas/PJOK_BS_KLS_VI.pdf filter=lfs diff=lfs merge=lfs -text
+Rag-Pipeline/Vektor[[:space:]]Database/Ipas/IPA_index.index filter=lfs diff=lfs merge=lfs -text
+Rag-Pipeline/Vektor[[:space:]]Database/Pancasila/PANCASILA_index.index filter=lfs diff=lfs merge=lfs -text
+Rag-Pipeline/Vektor[[:space:]]Database/Penjas/PENJAS_index.index filter=lfs diff=lfs merge=lfs -text

Rag-Pipeline/Vektor Database/Ipas/IPA_index.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45fdf20d00c4cbe679a6d00584f35b95942552ea7b67137779fdcd48c65b5403
+size 15818797

Rag-Pipeline/Vektor Database/Pancasila/PANCASILA_index.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:157e84a8342195a6e39eed3be5f244745fbf8221e92aba797c440067443e8afc
+size 12943405

Rag-Pipeline/Vektor Database/Penjas/PENJAS_index.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:295a1bef4823cf5d42f24aa8be1259962640e8f6beba94a69cd542ef84e231f9
+size 17891373

Rag-Pipeline/chunk.py

@@ -0,0 +1,45 @@
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+import glob
+import json
+import os
+
+
+folder_path = "D:\Webchatbot\Dataset\Penjas\Clean"
+file_paths = glob.glob(os.path.join(folder_path, "*.txt"))
+
+pages = []
+for path in file_paths:
+    with open(path, "r", encoding="utf-8") as f:
+        text = f.read()
+        pages.append(Document(page_content=text, metadata={"source": path}))
+
+print(f" Total file terbaca: {len(file_paths)}")
+
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size=300,
+    chunk_overlap=50,
+    separators=["\n\n", "\n", ".", " "]
+)
+
+documents = text_splitter.split_documents(pages)
+all_texts = [doc.page_content for doc in documents]
+
+
+output_dir = "D:\Webchatbot\Dataset\Penjas\Chunk"
+os.makedirs(output_dir, exist_ok=True)
+
+output_path = os.path.join(output_dir, "penjas_chunks.json")
+
+data_to_save = [
+    {"id": i + 1, "text": chunk}
+    for i, chunk in enumerate(all_texts)
+]
+
+with open(output_path, "w", encoding="utf-8") as f:
+    json.dump(data_to_save, f, ensure_ascii=False, indent=2)
+
+print(f"Hasil chunk disimpan ke: {os.path.abspath(output_path)}")
+
+for i, chunk in enumerate(all_texts[:3]):
+    print(f"\n--- Chunk {i+1} ---\n{chunk}")

Rag-Pipeline/cleans.py

@@ -0,0 +1,59 @@
+import os
+import re
+from pdfminer.high_level import extract_pages
+from pdfminer.layout import LTTextContainer
+
+
+def extract_and_clean_pdf(path: str, skip_pages: list[int] = None) -> list[str]:
+    skip_pages = skip_pages or []
+    cleaned_pages = []
+
+    for i, page_layout in enumerate(extract_pages(path), start=1):
+        if i in skip_pages:
+            print(f"Halaman {i} dilewati.")
+            continue
+
+        page_text = ""
+        for element in page_layout:
+            if isinstance(element, LTTextContainer):
+                page_text += element.get_text()
+
+        cleaned_text = clean_text(page_text)
+        cleaned_pages.append(cleaned_text)
+
+    print(f"\nTotal halaman diambil: {len(cleaned_pages)} halaman (dari {i} total halaman).")
+    return cleaned_pages
+
+
+def clean_text(text: str) -> str:
+    text = text.replace("\n", " ").replace("\t", " ")
+    text = re.sub(r'[^\x20-\x7EÀ-ÿ]', '', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'([a-z])([A-Z])', r'\1 \2', text)
+    text = re.sub(r'([a-z])([0-9])', r'\1 \2', text)
+    text = re.sub(r'([0-9])([a-zA-Z])', r'\1 \2', text)
+    text = re.sub(r'\s+([.,!?;:])', r'\1', text)
+    return text.strip()
+
+
+def save_cleaned_text(cleaned_pages: list[str], output_path: str):
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        for page in cleaned_pages:
+            f.write(page + "\n\n")
+
+    print(f"File teks berhasil disimpan ke:\n{output_path}")
+
+
+if __name__ == "__main__":
+    pdf_path = "D:\Webchatbot\Dataset\Penjas\PJOK_BS_KLS_V.pdf"
+    output_txt = "D:\Webchatbot\Dataset\Penjas\Clean\Penjas Kelas V.txt"
+
+    halaman_dihapus = []+ list(range(1,15)) + list(range(188,208))
+    hasil = extract_and_clean_pdf(pdf_path, skip_pages=halaman_dihapus)
+
+    if hasil:
+        save_cleaned_text(hasil, output_txt)
+    else:
+        print("Tidak ada halaman yang diekstrak.")

Rag-Pipeline/cleans2.py

@@ -0,0 +1,123 @@
+
+import os, re, sys
+from typing import List, Optional, Set
+import fitz  
+import pytesseract
+from PIL import Image
+from io import BytesIO
+
+PDF_PATH   = r"D:\Webchatbot\Dataset\Penjas\PJOK_BS_KLS_VI.pdf"
+OUTPUT_TXT = r"D:\Webchatbot\Dataset\Penjas\Clean\Penjas Kelas VI.txt"
+SKIP_PAGES = list(range(1, 22)) + list(range(200, 211)) + list(range(213, 226)) 
+TESSERACT_CMD = r"C:\Program Files\Tesseract-OCR\tesseract.exe" 
+OCR_LANG   = "ind+eng"
+DPI        = 300  
+
+
+if TESSERACT_CMD:
+    pytesseract.pytesseract.tesseract_cmd = TESSERACT_CMD
+
+
+URL_RE = re.compile(
+    r"(https?://\S+|www\.\S+|\b\S+\.(?:com|org|net|edu|gov|go|id|co)\S*)",
+    flags=re.IGNORECASE,
+)
+BAB_LINE_RE = re.compile(
+    r"^\s*(?:bab|BAB)\s*(?:[0-9]+|[IVXLCDM]+)\s*(?:[:\-–—]\s*.*)?\s*$"
+)
+BAB_PREFIX_RE = re.compile(
+    r"^\s*(?:bab|BAB)\s*(?:[0-9]+|[IVXLCDM]+)\s*(?:[:\-–—]\s*)?",
+    flags=re.IGNORECASE,
+)
+
+def clean_text(text: str) -> str:
+    text = URL_RE.sub("", text or "")
+
+    text = text.replace("\t", " ")
+    text = re.sub(r"[^\x09\x0A\x0D\x20-\x7EÀ-ÿ]", "", text)
+
+    cleaned_lines: List[str] = []
+    for raw_ln in text.splitlines():
+        ln = re.sub(r"\s+", " ", raw_ln).strip()
+        if not ln:
+            continue
+
+        if BAB_LINE_RE.match(ln):
+            continue
+
+        ln = BAB_PREFIX_RE.sub("", ln).strip()
+
+        if not ln:
+            continue
+
+        cleaned_lines.append(ln)
+
+    text_out = "\n".join(cleaned_lines).strip()
+    return text_out
+
+def pixmap_to_pil(pix: fitz.Pixmap) -> Image.Image:
+    if pix.alpha:
+        pix = fitz.Pixmap(fitz.csRGB, pix)
+    img_bytes = pix.tobytes("png")
+    return Image.open(BytesIO(img_bytes))
+
+def ocr_page(img: Image.Image, lang: str) -> str:
+    return clean_text(pytesseract.image_to_string(img, lang=lang))
+
+def main():
+    if not os.path.exists(PDF_PATH):
+        print(f"PDF tidak ditemukan: {PDF_PATH}")
+        sys.exit(1)
+
+    doc = fitz.open(PDF_PATH)
+    total = doc.page_count
+    skip: Set[int] = set(SKIP_PAGES or [])
+
+    zoom = DPI / 72.0
+    mat = fitz.Matrix(zoom, zoom)
+
+    results: List[str] = []
+    skipped = 0
+    kept = 0
+
+    print(f"[*] Total halaman: {total} | DPI render: {DPI}")
+    for page_num in range(1, total + 1): 
+        if page_num in skip:
+            skipped += 1
+            print(f"Halaman {page_num} dilewati.")
+            continue
+
+        page = doc.load_page(page_num - 1)
+        pix = page.get_pixmap(matrix=mat, alpha=False)
+        img = pixmap_to_pil(pix)
+
+        print(f"Halaman {page_num}: OCR …")
+        try:
+            txt = ocr_page(img, OCR_LANG)
+        except Exception as e:
+            print(f"[!] OCR gagal halaman {page_num}: {e}")
+            txt = ""
+
+        if txt.strip():
+            results.append(txt.strip())
+            kept += 1
+        else:
+            print(f"Halaman {page_num}: hasil kosong/pendek.")
+
+    doc.close()
+
+    os.makedirs(os.path.dirname(OUTPUT_TXT), exist_ok=True)
+    with open(OUTPUT_TXT, "w", encoding="utf-8") as f:
+        for t in results:
+            if not t.strip():
+                continue
+            f.write(t + "\n\n")
+
+    print("\nRingkasan:")
+    print(f"- Total halaman       : {total}")
+    print(f"- Dilewati (skip)     : {skipped}")
+    print(f"- Tersimpan (non-skip): {kept}")
+    print(f"[*] Output: {OUTPUT_TXT}")
+
+if __name__ == "__main__":
+    main()

Rag-Pipeline/embed.py

@@ -0,0 +1,54 @@
+import os, json
+import torch
+import torch.nn.functional as F
+import numpy as np
+from transformers import AutoTokenizer, AutoModel
+
+# ===== PATH =====
+JSON_PATH   = "D:\Webchatbot\Dataset\Penjas\Chunk\penjas_chunks.json"
+OUTPUT_DIR  = "D:\Webchatbot\Dataset\Penjas\Embedd"
+OUTPUT_NAME = "penjas_embeddings.npy"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+# ===== DEVICE =====
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print(f"Running on device: {device}")
+
+# ===== LOAD CHUNKS =====
+with open(JSON_PATH, "r", encoding="utf-8") as f:
+    data = json.load(f)
+texts = [item["text"] for item in data]
+print(f"Total chunk: {len(texts)}")
+
+# ===== MODEL =====
+MODEL_NAME = "intfloat/multilingual-e5-large"
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModel.from_pretrained(MODEL_NAME).to(device).eval()
+
+def mean_pooling(last_hidden_state, attention_mask):
+    mask = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()
+    summed = (last_hidden_state * mask).sum(dim=1)
+    counts = mask.sum(dim=1).clamp(min=1e-9)
+    return summed / counts
+
+@torch.no_grad()
+def get_embeddings(texts, batch_size=32, max_length=512):
+    embs = []
+    for i in range(0, len(texts), batch_size):
+        batch = [f"passage: {t}" for t in texts[i:i+batch_size]]  
+        inputs = tokenizer(batch, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to(device)
+        outputs = model(**inputs)
+        pooled = mean_pooling(outputs.last_hidden_state, inputs["attention_mask"])
+        pooled = F.normalize(pooled, p=2, dim=1)
+        embs.append(pooled.cpu())
+        if (i // batch_size) % 10 == 0:
+            print(f"Processed: {i+len(batch)}/{len(texts)}")
+    return torch.cat(embs, dim=0)
+
+embeddings = get_embeddings(texts, batch_size=32)
+print(f"Embeddings shape: {embeddings.shape}")
+
+# ===== SAVE NPY =====
+output_path = os.path.join(OUTPUT_DIR, OUTPUT_NAME)
+np.save(output_path, embeddings.numpy())
+print(f"Embeddings disimpan ke: {output_path}")

Rag-Pipeline/faissdb.py

@@ -0,0 +1,19 @@
+import numpy as np
+import faiss
+import os
+
+embeddings_path = "D:\Webchatbot\Dataset\Penjas\Embedd\penjas_embeddings.npy"
+
+output_dir = "D:\Webchatbot\Rag-Pipeline\Vektor Database\Penjas"
+
+embeddings_np = np.load(embeddings_path)
+print(f"Embeddings shape: {embeddings_np.shape}")
+
+dimension = embeddings_np.shape[1]  
+index = faiss.IndexFlatL2(dimension) 
+index.add(embeddings_np)
+print(f"Total vectors di FAISS: {index.ntotal}")
+
+faiss_index_path = os.path.join(output_dir, "PENJAS_index.index")
+faiss.write_index(index, faiss_index_path)
+print(f"FAISS index disimpan ke: {faiss_index_path}")