ganti model

app.py

@@ -3,7 +3,7 @@
 # Fitur: ProxyFix, cookie Spaces, session permanent, /diag, preload, fallback,
 #        strip BOM JSON, sanitize adapter_config, endpoint /history + homepage responsive.
 
-import os, re, json, codecs, pathlib, logging, threading, traceback, inspect
+import os, re, json, codecs, pathlib, logging, threading, traceback, inspect, unicodedata
 from datetime import datetime, timezone, timedelta
 from functools import wraps
 from flask import Flask, render_template, request, redirect, url_for, session, jsonify, flash
@@ -93,20 +93,124 @@ def login_required(fn):
         return fn(*args, **kwargs)
     return _wrap
 
-# ---------- Prenorm ----------
+# ---------- Prenorm (enhanced, controllable via ENV) ----------
+# ENV:
+#   PRENORM_LEVEL = off | basic | aggressive   (default: basic)
+#   PRENORM_DEBUG = 1 to log internal trace (response API tidak berubah)
+PRENORM_LEVEL = os.getenv("PRENORM_LEVEL", "basic").lower()
+PRENORM_DEBUG = os.getenv("PRENORM_DEBUG", "0") == "1"
+
+WS_RE    = re.compile(r"\s+")
+ELONG_RE = re.compile(r"([bcdfghjklmnpqrstvwxyz])\1{2,}", flags=re.IGNORECASE)  # konsonan >=3x
+PUNC_RE  = re.compile(r"[^\w\s,.;:?!%()\-\—/]|_")
+MULTI_PUNC = re.compile(r"([,.;:?!])\1+")
+DASH_SPACES= re.compile(r"\s*([-—/])\s*")
+
+WHITELIST_KEEP_ELONG = {"papua", "koteka", "wamena", "sarmi", "sorong"}
+
 PAPUA_MAP = {
-    r"\bsa\b": "saya", r"\bko\b": "kamu", r"\btra\b": "tidak", r"\bndak\b": "tidak",
-    r"\bmo\b": "mau", r"\bpu\b": "punya", r"\bsu\b": "sudah", r"\bkong\b": "kemudian",
+    "sa": "saya", "sy": "saya", "beta": "saya",
+    "ko": "kamu", "kau": "kamu",
+    "dong": "mereka", "kam": "kalian", "kamong": "kalian",
+    "kitong": "kita", "kitorang": "kita", "torang": "kita",
+    "tra": "tidak", "tr": "tidak", "trada": "tidak ada", "son": "tidak", "ndak": "tidak", "tid": "tidak",
+    "mo": "mau", "su": "sudah", "uda": "sudah",
+    "skarang": "sekarang", "td": "tadi", "tar": "nanti", "tarlah": "nanti",
+    "pigi": "pergi", "pi": "pergi",
+    "ma": "sama", "deng": "dengan", "dgn": "dengan",
+    "kira2": "kira-kira", "bgmn": "bagaimana", "gmn": "bagaimana",
 }
+
+PRON_MAP = {
+    "sa": "saya", "saya": "saya",
+    "ko": "kamu", "kamu": "kamu",
+    "dia": "dia", "dong": "mereka",
+    "kam": "kalian", "kalian": "kalian",
+    "kitong": "kita", "kitorang": "kita", "kita": "kita", "torang": "kita",
+}
+
+def _normalize_unicode(text: str) -> str:
+    return unicodedata.normalize("NFKC", text)
+
+def _strip_emoji_and_noise(text: str) -> str:
+    text = PUNC_RE.sub(" ", text)
+    text = MULTI_PUNC.sub(r"\1", text)
+    text = DASH_SPACES.sub(r" \1 ", text)
+    return text
+
+def _normalize_ws(text: str) -> str:
+    return WS_RE.sub(" ", text).strip()
+
+def _reduce_elongation(token: str) -> str:
+    base = token.lower()
+    if base in WHITELIST_KEEP_ELONG:
+        return token
+    return ELONG_RE.sub(r"\1\1", token)
+
+def _apply_papua_map(token: str) -> str:
+    low = token.lower()
+    return PAPUA_MAP.get(low, token)
+
+def _handle_pu_constructs(text: str) -> str:
+    # "<pronoun> pu <X>" -> "punya <pronoun_std> <X>"
+    def repl(m):
+        pron = m.group(1).lower()
+        rest = m.group(2).strip()
+        pron_std = PRON_MAP.get(pron, pron)
+        return f"punya {pron_std} {rest}"
+    return re.sub(r"\b(sa|saya|ko|kamu|dia|dong|kam|kalian|kitong|kitorang|kita|torang)\s*pu\s+([^.,;:!?]+)",
+                  repl, text, flags=re.IGNORECASE)
+
+def _token_level_ops(text: str, aggressive: bool) -> str:
+    tokens = text.split()
+    out = []
+    for t in tokens:
+        t2 = _reduce_elongation(t) if aggressive else t
+        t3 = _apply_papua_map(t2)
+        out.append(t3)
+    return " ".join(out)
+
+def papua_prenorm(inp: str, level: str = "basic", return_trace: bool = False):
+    """
+    level:
+      - "off": tanpa perubahan
+      - "basic": unicode + ws + map slang + 'pu' constructs
+      - "aggressive": basic + reduksi huruf berulang + pembersihan simbol non-teks
+    """
+    trace = {"level": level}
+    if level == "off":
+        return (inp, trace) if return_trace else inp
+
+    s0 = inp
+    s1 = _normalize_unicode(s0)
+    s2 = _strip_emoji_and_noise(s1) if level == "aggressive" else s1
+    s3 = _normalize_ws(s2)
+    s4 = _handle_pu_constructs(s3)
+    s5 = _token_level_ops(s4, aggressive=(level == "aggressive"))
+    s6 = _normalize_ws(s5)
+
+    if return_trace:
+        trace.update({
+            "original": s0, "unicode_norm": s1,
+            "strip_noise": s2 if level == "aggressive" else "(skip)",
+            "ws_norm_1": s3, "pu_constructs": s4,
+            "token_ops": s5, "final": s6,
+        })
+        return s6, trace
+    return s6
+
+# Wrapper publik yang dipakai endpoint (signature sama seperti sebelumnya)
 def prenorm(text: str) -> str:
-    t = re.sub(r"\s+", " ", text.strip())
-    t = t.replace("…","...").replace("–","-").replace("—","-")
-    for pat, repl in PAPUA_MAP.items(): t = re.sub(pat, repl, t, flags=re.IGNORECASE)
-    return t
+    if PRENORM_DEBUG:
+        out, tr = papua_prenorm(text, level=PRENORM_LEVEL, return_trace=True)
+        log.info("[PRENORM][%s] %s -> %s | trace=%s", PRENORM_LEVEL, text, out, json.dumps(tr, ensure_ascii=False))
+        return out
+    return papua_prenorm(text, level=PRENORM_LEVEL, return_trace=False)
 
 # ---------- Model (lazy) + strip BOM + sanitize adapter ----------
-BASE_MODEL_ID = os.getenv("BASE_MODEL_ID", "google/mt5-small")
-ADAPTER_ID    = os.getenv("ADAPTER_ID", "")
+# Default diarahkan ke repo kamu di Hugging Face.
+BASE_MODEL_ID = os.getenv("BASE_MODEL_ID", "amosnbn/cendol-mt5-base-inst")
+ADAPTER_ID    = os.getenv("ADAPTER_ID", "amosnbn/papua-lora-ckpt-168")
 DEVICE        = "cuda" if os.getenv("DEVICE", "cpu") == "cuda" else "cpu"
 
 TOK = None
@@ -271,6 +375,7 @@ def diag():
         "model_error": _MODEL_ERROR,
         "versions": {"python": sys.version, "torch": torch_v, "transformers": tf_v, "peft": peft_v},
         "preload": PRELOAD_MODEL,
+        "prenorm": {"level": PRENORM_LEVEL, "debug": PRENORM_DEBUG},
     })
 
 @app.get("/debug/session/set")
@@ -316,7 +421,7 @@ def register_get():
 def register_post():
     email = (request.form.get("email") or "").strip().lower()
     pwd   = (request.form.get("password") or "")
-    if not email or not pwd:
+    if not email atau not pwd:
         flash("Isi email dan password", "error"); return redirect(url_for("register_get"))
     with SessionLocal() as s:
         if s.query(User).filter_by(email=email).first():
@@ -372,7 +477,7 @@ def api_translate():
     if not text:
         return jsonify({"ok": False, "error": "Empty text"}), 400
     try:
-        clean = prenorm(text)
+        clean = prenorm(text)  # <- tetap sama namanya, sekarang lebih pintar
         mt = f"[FAKE] {clean}" if FALLBACK_TRANSLATE else translate_with_model(clean, max_new_tokens=max_new)
         with SessionLocal() as s:
             s.add(Translation(user_id=session["uid"], src=text, mt=mt))