19

app.py

@@ -1,35 +1,41 @@
 # app.py
-# PapuaTranslate — Flask + SQLAlchemy (Supabase/SQLite) + mT5-LoRA (lazy) + diag + preload
-import os, re, logging, threading, traceback
+# PapuaTranslate — Flask + SQLAlchemy (Supabase/SQLite) + mT5/LoRA (lazy)
+# Fitur: ProxyFix, cookie Spaces, session permanent, /diag, preload, fallback, strip BOM JSON.
+
+import os, re, json, codecs, pathlib, logging, threading, traceback
 from datetime import datetime, timezone, timedelta
 from functools import wraps
 from flask import Flask, render_template, request, redirect, url_for, session, jsonify, flash
 from werkzeug.middleware.proxy_fix import ProxyFix
+from huggingface_hub import snapshot_download
 
-# ===== Logging =====
+# ========== Logging ==========
 logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
 log = logging.getLogger("papua-app")
 
-# ===== Flask / Template =====
+# ========== Flask ==========
 app = Flask(__name__, template_folder="frontend", static_folder="static")
+
+# Trust reverse proxy di HF supaya proto/host benar (perlu untuk cookie Secure)
 app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1)
 
+# Cookie/session aman untuk iframe (HF Spaces) dan tab langsung hf.space
 app.config.update(
     SECRET_KEY=os.getenv("SECRET_KEY", "dev-secret-change-me"),
     SESSION_COOKIE_NAME="hfspace_session",
-    SESSION_COOKIE_SAMESITE="None",
-    SESSION_COOKIE_SECURE=True,
+    SESSION_COOKIE_SAMESITE="None",   # penting: iframe = third-party context
+    SESSION_COOKIE_SECURE=True,       # wajib True jika SAMESITE=None
     SESSION_COOKIE_HTTPONLY=True,
     SESSION_COOKIE_PATH="/",
     PREFERRED_URL_SCHEME="https",
 )
 app.permanent_session_lifetime = timedelta(hours=8)
 
-# ===== Feature flags via ENV =====
-PRELOAD_MODEL       = os.getenv("PRELOAD_MODEL", "true").lower() in ("1","true","yes")
-FALLBACK_TRANSLATE  = os.getenv("FALLBACK_TRANSLATE", "false").lower() in ("1","true","yes")
+# ========== Feature Flags via ENV ==========
+PRELOAD_MODEL      = os.getenv("PRELOAD_MODEL", "true").lower() in ("1","true","yes")
+FALLBACK_TRANSLATE = os.getenv("FALLBACK_TRANSLATE", "false").lower() in ("1","true","yes")
 
-# ===== DB: SQLAlchemy =====
+# ========== Database (SQLAlchemy) ==========
 from sqlalchemy import create_engine, Column, Integer, Text, DateTime, ForeignKey, func
 from sqlalchemy.orm import declarative_base, sessionmaker, scoped_session, relationship
 
@@ -38,10 +44,12 @@ if not DATABASE_URL:
     DATABASE_URL = "sqlite:////tmp/app.db"
     log.warning("[DB] DATABASE_URL tidak diset; pakai SQLite /tmp/app.db")
 else:
+    # normalisasi skema ke psycopg2
     if DATABASE_URL.startswith("postgres://"):
         DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql+psycopg2://", 1)
     elif DATABASE_URL.startswith("postgresql://"):
         DATABASE_URL = DATABASE_URL.replace("postgresql://", "postgresql+psycopg2://", 1)
+    # tambahkan sslmode kalau belum ada
     if DATABASE_URL.startswith("postgresql+psycopg2") and "sslmode=" not in DATABASE_URL:
         sep = "&" if "?" in DATABASE_URL else "?"
         DATABASE_URL = f"{DATABASE_URL}{sep}sslmode=require"
@@ -72,7 +80,7 @@ try:
 except Exception as e:
     log.exception("[DB] init error: %s", e)
 
-# ===== Auth helpers =====
+# ========== Auth Helpers ==========
 from werkzeug.security import generate_password_hash, check_password_hash
 def set_password(user: User, raw: str): user.pass_hash = generate_password_hash(raw)
 def verify_password(user: User, raw: str) -> bool:
@@ -87,7 +95,7 @@ def login_required(fn):
         return fn(*args, **kwargs)
     return _wrap
 
-# ===== Prenorm =====
+# ========== Prenorm (heuristik ringan) ==========
 PAPUA_MAP = {
     r"\bsa\b": "saya", r"\bko\b": "kamu", r"\btra\b": "tidak", r"\bndak\b": "tidak",
     r"\bmo\b": "mau", r"\bpu\b": "punya", r"\bsu\b": "sudah", r"\bkong\b": "kemudian",
@@ -98,9 +106,9 @@ def prenorm(text: str) -> str:
     for pat, repl in PAPUA_MAP.items(): t = re.sub(pat, repl, t, flags=re.IGNORECASE)
     return t
 
-# ===== Model (lazy) =====
-BASE_MODEL_ID = os.getenv("BASE_MODEL_ID", "google/mt5-small")  # kecil dulu untuk uji
-ADAPTER_ID    = os.getenv("ADAPTER_ID", "")                     # kosongkan dulu
+# ========== Model (lazy) + Strip BOM ==========
+BASE_MODEL_ID = os.getenv("BASE_MODEL_ID", "google/mt5-small")  # mulai kecil dulu untuk uji cepat
+ADAPTER_ID    = os.getenv("ADAPTER_ID", "")                     # kosongkan dulu; isi nanti
 DEVICE        = "cuda" if os.getenv("DEVICE", "cpu") == "cuda" else "cpu"
 
 TOK = None
@@ -109,20 +117,58 @@ _MODEL_LOCK = threading.Lock()
 _MODEL_READY = False
 _MODEL_ERROR = None
 
+def _strip_bom_in_dir(root_dir: str):
+    """Hapus BOM dari semua *.json (UTF-8-sig) agar tidak memicu JSONDecodeError."""
+    root = pathlib.Path(root_dir)
+    for p in root.rglob("*.json"):
+        try:
+            with codecs.open(p, "r", encoding="utf-8-sig") as f:
+                data = json.load(f)
+            with open(p, "w", encoding="utf-8") as f:
+                json.dump(data, f, ensure_ascii=False, indent=2)
+            log.info(f"[BOM] stripped: {p}")
+        except Exception as e:
+            # Jangan gagal hanya karena satu file
+            log.warning(f"[BOM] skip {p}: {e}")
+
 def _load_model():
+    """Download ke /tmp, strip BOM, lalu load dari path lokal."""
     global TOK, MODEL, _MODEL_READY, _MODEL_ERROR
     try:
-        log.info("[MODEL] loading base=%s adapter=%s", BASE_MODEL_ID, ADAPTER_ID or "-")
-        # import di sini agar error import terlihat di /diag
+        log.info("[MODEL] downloading base=%s adapter=%s", BASE_MODEL_ID, ADAPTER_ID or "-")
+
+        base_dir = snapshot_download(
+            repo_id=BASE_MODEL_ID,
+            local_dir="/tmp/hf_base",
+            local_dir_use_symlinks=False,
+            allow_patterns=None,
+        )
+        _strip_bom_in_dir(base_dir)
+
+        adapter_dir = None
+        if ADAPTER_ID:
+            adapter_dir = snapshot_download(
+                repo_id=ADAPTER_ID,
+                local_dir="/tmp/hf_adapter",
+                local_dir_use_symlinks=False,
+                allow_patterns=None,
+            )
+            _strip_bom_in_dir(adapter_dir)
+
+        # Import di sini agar error versi lib ketangkep di /diag
         import torch
         from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
         from peft import PeftModel
 
-        TOK = AutoTokenizer.from_pretrained(BASE_MODEL_ID)
-        base = AutoModelForSeq2SeqLM.from_pretrained(BASE_MODEL_ID)
+        log.info("[MODEL] loading tokenizer from %s", base_dir)
+        TOK = AutoTokenizer.from_pretrained(base_dir)
 
-        if ADAPTER_ID:
-            MODEL = PeftModel.from_pretrained(base, ADAPTER_ID)
+        log.info("[MODEL] loading base model from %s", base_dir)
+        base = AutoModelForSeq2SeqLM.from_pretrained(base_dir)
+
+        if adapter_dir:
+            log.info("[MODEL] attaching adapter from %s", adapter_dir)
+            MODEL = PeftModel.from_pretrained(base, adapter_dir)
         else:
             MODEL = base
 
@@ -149,11 +195,11 @@ def translate_with_model(text: str, max_new_tokens: int = 48) -> str:
     if not _MODEL_READY or m is None:
         raise RuntimeError(f"Model not ready: {_MODEL_ERROR or 'unknown error'}")
 
-    # guard panjang input agar tidak OOM
+    # Guard panjang input supaya aman
     enc = tok([text], return_tensors="pt", truncation=True, max_length=256)
     enc = {k: v.to(DEVICE) for k, v in enc.items()}
 
-    log.info("[GEN] start (len=%d)", int(enc["input_ids"].shape[-1]))
+    log.info("[GEN] start (tok_len=%d)", int(enc["input_ids"].shape[-1]))
     out = m.generate(
         **enc,
         max_new_tokens=int(max_new_tokens),
@@ -165,7 +211,7 @@ def translate_with_model(text: str, max_new_tokens: int = 48) -> str:
     log.info("[GEN] done")
     return tok.decode(out[0], skip_special_tokens=True)
 
-# preload (opsional) supaya request pertama nggak lama
+# Preload supaya request pertama nggak lama/timeout
 def _preload_thread():
     try:
         _load_model()
@@ -175,7 +221,7 @@ def _preload_thread():
 if PRELOAD_MODEL:
     threading.Thread(target=_preload_thread, daemon=True).start()
 
-# ===== Utils / logging =====
+# ========== Utils / Logging ==========
 @app.before_request
 def _log_req():
     if request.path not in ("/health", "/ping", "/favicon.ico"):
@@ -186,7 +232,7 @@ def _err(e):
     log.exception("Unhandled error")
     return "Internal Server Error", 500
 
-# ===== Debug / Diag =====
+# ========== Debug & Diag ==========
 @app.get("/diag")
 def diag():
     import sys
@@ -221,7 +267,7 @@ def dbg_set():
 def dbg_get():
     return {"uid": session.get("uid"), "email": session.get("email")}
 
-# ===== Routes =====
+# ========== Routes ==========
 @app.get("/health")
 @app.get("/ping")
 def health():
@@ -299,22 +345,21 @@ def api_translate():
     try:
         clean = prenorm(text)
 
-        # Jika ingin memastikan alur UI/DB dulu tanpa model:
         if FALLBACK_TRANSLATE:
             mt = f"[FAKE] {clean}"
         else:
             mt = translate_with_model(clean, max_new_tokens=max_new)
 
-        # simpan riwayat
+        # Simpan riwayat
         with SessionLocal() as s:
             s.add(Translation(user_id=session["uid"], src=text, mt=mt))
             s.commit()
 
-        return jsonify({"ok": True, "result": mt})
+        # Kompatibel dengan frontend kamu (pakai j.mt)
+        return jsonify({"ok": True, "mt": mt})
     except Exception as e:
         log.error("[API] translate error: %s", e)
         log.error(traceback.format_exc())
-        # kirim error yang lebih informatif
         return jsonify({"ok": False, "error": f"{type(e).__name__}: {e}"}), 500
 
 if __name__ == "__main__":