Update rag_pipeline.py

rag_pipeline.py

@@ -6,24 +6,23 @@ from pathlib import Path
 
 import faiss
 import numpy as np
+from huggingface_hub import hf_hub_download
 
 from providers import embed, generate, rerank, qa_extract
 
-try:
-    from huggingface_hub import hf_hub_download
-except Exception:
-    hf_hub_download = None  # requirements'ta yoksa problem değil
-
 # =========================
-# Varsayılan yollar
+# Dosya yolları ve sabitler
 # =========================
-VSTORE_DIR  = "vectorstore"
-FAISS_FILE  = "index.faiss"
-META_JSONL  = "meta.jsonl"
 
-# =========================
+# Çalışma kökü (Space içinde /home/user/app)
+APP_ROOT = Path.cwd()
+
+# Varsayılan vektör klasörü
+VSTORE_DIR = APP_ROOT / "vectorstore"
+FAISS_FILE = "index.faiss"
+META_JSONL = "meta.jsonl"
+
 # Hız / kalite ayarları
-# =========================
 TOP_K_DEFAULT     = 4
 FETCH_K_DEFAULT   = 16
 HNSW_EFSEARCH     = 32
@@ -40,11 +39,12 @@ W_LEXICAL         = 0.15
 # =========================
 # Kural-tabanlı yardımcılar
 # =========================
+
 DATE_RX = re.compile(
     r"\b(\d{1,2}\s+(Ocak|Şubat|Mart|Nisan|Mayıs|Haziran|Temmuz|Ağustos|Eylül|Ekim|Kasım|Aralık)\s+\d{3,4}"
     r"|\d{1,2}\.\d{1,2}\.\d{2,4}"
     r"|\d{4})\b",
-    flags=re.IGNORECASE
+    flags=re.IGNORECASE,
 )
 DEATH_KEYS = ["öldü", "vefat", "hayatını kaybet", "ölümü", "ölüm"]
 FOUND_KEYS = ["kuruldu", "kuruluş", "kurulmuştur", "kuruluşu", "kuruluş tarihi"]
@@ -63,6 +63,7 @@ def _extract_fact_sentence(query: str, hits: List[Dict]) -> Tuple[str, str]:
         keylist = FOUND_KEYS
     else:
         keylist = DEATH_KEYS + FOUND_KEYS
+
     for h in hits:
         sents = _split_sentences(h.get("text", ""))
         for s in sents:
@@ -70,19 +71,29 @@ def _extract_fact_sentence(query: str, hits: List[Dict]) -> Tuple[str, str]:
                 return s, h.get("source", "")
     return "", ""
 
+# =========================
+# İsim normalizasyonu
+# =========================
+
 NAME_RX = re.compile(r"\b([A-ZÇĞİIÖŞÜ][a-zçğıiöşü]+(?:\s+[A-ZÇĞİIÖŞÜ][a-zçğıiöşü]+){0,3})\b")
 
 def _expand_named_span(answer: str, hits: List[Dict]) -> str:
     ans = (answer or "").strip()
     if not ans or len(ans.split()) > 2:
         return ans
+
     ans_low = ans.lower()
-    preferred_aliases = ["Mustafa Kemal Atatürk", "Sabiha Gökçen", "İsmet İnönü"]
+    preferred_aliases = [
+        "Mustafa Kemal Atatürk",
+        "Sabiha Gökçen",
+        "İsmet İnönü",
+    ]
     for h in hits:
         text = h.get("text", "")
         for alias in preferred_aliases:
-            if ans_low in alias.lower() and alias in text:
+            if alias.lower().find(ans_low) != -1 and alias in text:
                 return alias
+
     best = ans
     for h in hits:
         for sent in _split_sentences(h.get("text", "")):
@@ -90,99 +101,86 @@ def _expand_named_span(answer: str, hits: List[Dict]) -> str:
                 continue
             for m in NAME_RX.finditer(sent):
                 cand = m.group(1).strip()
-                if ans_low in cand.lower():
-                    if len(cand) >= len(best) and any(ch.islower() for ch in cand):
-                        best = cand if len(cand.split()) >= len(best.split()) else best
+                if ans_low in cand.lower() and any(ch.islower() for ch in cand):
+                    if len(cand.split()) >= len(best.split()):
+                        best = cand
     return best
 
-# -------------------------
-# Hugging Face Hub fallback
-# -------------------------
-def _hub_download_if_possible(rel_path: str) -> str:
-    if hf_hub_download is None:
-        return ""
-    repo_id = os.getenv("HF_SPACE_ID") or os.getenv("SPACE_ID")
-    if not repo_id:
-        return ""
-    try:
-        return hf_hub_download(repo_id=repo_id, filename=rel_path, repo_type="space")
-    except Exception:
-        return ""
-
 # =========================
-# **BURASI KRİTİK**: Vektör deposunu yükle
+# Vektör deposunu yükle
 # =========================
-def load_vectorstore(vstore_dir: str = "vectorstore"):
+
+def _ensure_local_file(rel_path: str) -> Path:
     """
-    1) Aşağıdaki aday klasörlerde dosyaları sırayla ara:
-       - verilen vstore_dir
-       - çalışma dizini / vstore_dir
-       - bu dosyanın 1 üst klasörü (repo kökü) / vstore_dir
-       - /home/user / vstore_dir  (HF Spaces tipik kök)
-    2) Yerelde bulunamazsa aynı Space deposundan (repo_type='space') indir.
+    Eğer ./vectorstore/<dosya> yoksa, aynı Space reposundan indir.
+    REPO_ID otomatik olarak SPACE_ID (owner/space) ile doldurulur.
     """
-    # Aday dizinler
-    candidates = []
-    try:
-        this_dir = Path(__file__).resolve().parent
-        repo_root = this_dir.parent
-    except Exception:
-        this_dir = Path.cwd()
-        repo_root = Path.cwd()
-
-    candidates += [
-        Path(vstore_dir),
-        Path.cwd() / vstore_dir,
-        repo_root / vstore_dir,
-        Path("/home/user") / vstore_dir,         # HF Spaces common
-        Path("/home/user/app") / vstore_dir,     # bazı runner'lar
-    ]
+    local_path = APP_ROOT / rel_path
+    if local_path.exists():
+        return local_path
 
-    faiss_path = ""
-    meta_path  = ""
-    for base in candidates:
-        f = base / FAISS_FILE
-        m = base / META_JSONL
-        if f.exists() and m.exists():
-            faiss_path, meta_path = str(f), str(m)
-            break
+    local_path.parent.mkdir(parents=True, exist_ok=True)
 
-    # Hub fallback
-    if not (faiss_path and meta_path):
-        hub_f = _hub_download_if_possible(f"{VSTORE_DIR}/{FAISS_FILE}")
-        hub_m = _hub_download_if_possible(f"{VSTORE_DIR}/{META_JSONL}")
-        if hub_f and hub_m and os.path.exists(hub_f) and os.path.exists(hub_m):
-            faiss_path, meta_path = hub_f, hub_m
+    repo_id = os.environ.get("REPO_ID") or os.environ.get("SPACE_ID")
+    if not repo_id:
+        raise FileNotFoundError(
+            f"'{rel_path}' bulunamadı ve REPO_ID/SPACE_ID çevre değişkeni setli değil."
+        )
 
-    if not (faiss_path and meta_path):
+    try:
+        # hf_hub_download hedef dosyayı tmp'e indirir; biz local_path'e kopyalarız
+        downloaded = hf_hub_download(
+            repo_id=repo_id,
+            filename=rel_path,  # örn: 'vectorstore/index.faiss'
+            local_dir=str(APP_ROOT),
+            local_dir_use_symlinks=False,
+        )
+        return Path(downloaded)
+    except Exception as e:
         raise FileNotFoundError(
-            "'vectorstore/index.faiss' indirilemedi veya bulunamadı. "
-            "Lütfen bu dosyayı Space deposunda 'vectorstore/' klasörüne yükleyin."
+            f"'{rel_path}' indirilemedi veya bulunamadı. Lütfen bu dosyayı Space deposunda "
+            f"'vectorstore/' klasörüne yükleyin. Ayrıntı: {e}"
         )
 
-    index = faiss.read_index(faiss_path)
+def load_vectorstore(vstore_dir: str | Path = VSTORE_DIR) -> Tuple[faiss.Index, List[Dict]]:
+    """
+    Önce yerelden yükler; yoksa aynı Space’ten indirip kaydeder.
+    app.py içinden hiçbir ek değişiklik yapmadan çalışır.
+    """
+    vstore_dir = Path(vstore_dir)
+    faiss_rel  = str(Path("vectorstore") / FAISS_FILE)
+    meta_rel   = str(Path("vectorstore") / META_JSONL)
+
+    # Yereli sağlama/alma
+    faiss_path = _ensure_local_file(faiss_rel)
+    meta_path  = _ensure_local_file(meta_rel)
+
+    # FAISS index
+    index = faiss.read_index(str(faiss_path))
     try:
         index.hnsw.efSearch = HNSW_EFSEARCH
     except Exception:
         pass
 
+    # Kayıtları oku
     records: List[Dict] = []
     with open(meta_path, "r", encoding="utf-8") as f:
         for line in f:
             obj = json.loads(line)
-            md = obj.get("metadata", {}) or {}
             records.append({
                 "text": obj.get("text", ""),
-                "title": md.get("title", ""),
-                "source": md.get("source", ""),
+                "title": (obj.get("metadata") or {}).get("title", ""),
+                "source": (obj.get("metadata") or {}).get("source", ""),
             })
+
     if not records:
         raise RuntimeError("meta.jsonl boş görünüyor.")
     return index, records
 
 # =========================
-# Arama + yeniden sıralama
+# Anahtar kelime çıkarımı + lexical puan
 # =========================
+
 _CAP_WORD = re.compile(r"\b([A-ZÇĞİIÖŞÜ][a-zçğıiöşü]+(?:\s+[A-ZÇĞİIÖŞÜ][a-zçğıiöşü]+)*)\b")
 
 def _keywords_from_query(q: str) -> List[str]:
@@ -203,27 +201,36 @@ def _lexical_overlap(q_tokens: List[str], text: str) -> float:
     denom = len(qset) or 1
     return inter / denom
 
+# =========================
+# Retrieval + (koşullu) rerank
+# =========================
+
 @lru_cache(maxsize=256)
 def _cached_query_vec(e5_query: str) -> np.ndarray:
     v = embed([e5_query]).astype("float32")
     return v
 
-def search_chunks(query: str, index: faiss.Index, records: List[Dict],
-                  top_k: int = TOP_K_DEFAULT, fetch_k: int = FETCH_K_DEFAULT) -> List[Dict]:
+def search_chunks(
+    query: str,
+    index: faiss.Index,
+    records: List[Dict],
+    top_k: int = TOP_K_DEFAULT,
+    fetch_k: int = FETCH_K_DEFAULT,
+) -> List[Dict]:
     q = (query or "").strip()
-    q_e5 = "query: " + q
-    q_vec = _cached_query_vec(q_e5)
+    q_vec = _cached_query_vec("query: " + q)
     faiss.normalize_L2(q_vec)
+
     scores, idxs = index.search(q_vec, fetch_k)
 
     pool: List[Dict] = []
     for i, s in zip(idxs[0], scores[0]):
         if 0 <= i < len(records):
-            md = records[i]
+            r = records[i]
             pool.append({
-                "text": md["text"],
-                "title": md.get("title", ""),
-                "source": md.get("source", ""),
+                "text": r["text"],
+                "title": r.get("title", ""),
+                "source": r.get("source", ""),
                 "score_vec": float(s),
             })
     if not pool:
@@ -238,31 +245,33 @@ def search_chunks(query: str, index: faiss.Index, records: List[Dict],
         lex = _lexical_overlap(q_tokens_lower, p["text"]) * W_LEXICAL
         p["score_boosted"] = p["score_vec"] + title_boost + lex
 
-    pool_by_boost = sorted(pool, key=lambda x: x["score_boosted"], reverse=True)
+    pool.sort(key=lambda x: x["score_boosted"], reverse=True)
 
-    if len(pool_by_boost) >= 2:
-        top1, top2 = pool_by_boost[0]["score_boosted"], pool_by_boost[1]["score_boosted"]
+    if len(pool) >= 2:
+        top1, top2 = pool[0]["score_boosted"], pool[1]["score_boosted"]
     else:
-        top1, top2 = pool_by_boost[0]["score_boosted"], 0.0
+        top1, top2 = pool[0]["score_boosted"], 0.0
     do_rerank = not (top1 >= HIGH_SCORE_THRES and (top1 - top2) >= MARGIN_THRES)
 
     if do_rerank:
-        rs = rerank(q, [p["text"] for p in pool_by_boost])
-        for p, r in zip(pool_by_boost, rs):
+        rs = rerank(q, [p["text"] for p in pool])
+        for p, r in zip(pool, rs):
             p["score_rerank"] = float(r)
-        pool_by_boost.sort(key=lambda x: (x.get("score_rerank", 0.0), x["score_boosted"]), reverse=True)
+        pool.sort(key=lambda x: (x.get("score_rerank", 0.0), x["score_boosted"]), reverse=True)
 
-    return pool_by_boost[:top_k]
+    return pool[:top_k]
 
 # =========================
 # LLM bağlamı ve kaynak listesi
 # =========================
+
 def _format_sources(hits: List[Dict]) -> str:
     seen, urls = set(), []
     for h in hits:
         u = (h.get("source") or "").strip()
         if u and u not in seen:
-            urls.append(u); seen.add(u)
+            urls.append(u)
+            seen.add(u)
     return "\n".join(f"- {u}" for u in urls) if urls else "- (yok)"
 
 def _llm_context(hits: List[Dict], limit: int = CTX_CHAR_LIMIT) -> str:
@@ -271,13 +280,20 @@ def _llm_context(hits: List[Dict], limit: int = CTX_CHAR_LIMIT) -> str:
         block = f"[{i}] {h.get('title','')} — {h.get('source','')}\n{h.get('text','')}"
         if total + len(block) > limit:
             break
-        ctx.append(block); total += len(block)
+        ctx.append(block)
+        total += len(block)
     return "\n\n---\n\n".join(ctx)
 
 # =========================
-# Nihai cevap
+# Nihai cevap (kural → QA → LLM → fallback)
 # =========================
-def generate_answer(query: str, index: faiss.Index, records: List[Dict], top_k: int = TOP_K_DEFAULT) -> str:
+
+def generate_answer(
+    query: str,
+    index: faiss.Index,
+    records: List[Dict],
+    top_k: int = TOP_K_DEFAULT,
+) -> str:
     hits = search_chunks(query, index, records, top_k=top_k)
     if not hits:
         return "Bilgi bulunamadı."
@@ -319,10 +335,12 @@ def generate_answer(query: str, index: faiss.Index, records: List[Dict], top_k:
         f"Soru:\n{query}\n\nBağlam:\n{context}\n\nYanıtı 1-2 cümlede ver."
     )
     llm_ans = (generate(prompt) or "").strip()
+
     if (not llm_ans) or ("yapılandırılmadı" in llm_ans.lower()):
         text = hits[0].get("text", "")
         first = re.split(r"(?<=[.!?])\s+", text.strip())[:2]
         llm_ans = " ".join(first).strip() or "Verilen bağlamda bu sorunun cevabı bulunmamaktadır."
+
     if "Kaynaklar:" not in llm_ans:
         llm_ans += "\n\nKaynaklar:\n" + _format_sources(hits)
     return llm_ans
@@ -330,6 +348,7 @@ def generate_answer(query: str, index: faiss.Index, records: List[Dict], top_k:
 # =========================
 # Hızlı test
 # =========================
+
 if __name__ == "__main__":
     idx, recs = load_vectorstore()
     for q in [