Update app.py

app.py

@@ -1,16 +1,20 @@
 # JusticeAI Backend — Upgraded & Integrated (Backend-only; does NOT create or overwrite frontend)
 #
-# Summary of recent changes applied in this file:
-# - Use the user's local emojis.py when available (falls back only if missing).
-# - Removed langdetect import and rely on a conservative heuristic (detect_language_safe).
-# - Prefer a local language model (language.py or language.bin) for translations; fall back to Helsinki-NLP via transformers if needed.
-# - Ensure /chat and all retrieval/refinement operate strictly within the resolved topic (no cross-topic lookup/updates).
-# - Stop joining sentences into run-on paragraphs. dedupe_sentences preserves sentences as separate lines.
-# - Enhance emoji understanding: detect emojis in user input, adjust mood detection using emojis, and apply safer rules for when to append or echo emojis in replies.
-# - Minor safety: never append emojis to replies that already contain them or when moderation flags high toxicity.
+# This is the updated app.py requested: it prefers a local language model (language.py or language.bin),
+# enforces strict topic scoping, preserves sentence boundaries (no run-on joining), understands and
+# reasons about emojis, and uses the provided emojis.py when present.
 #
-# Notes: keep testing with your local emojis.py and language.* model. If your language module exposes specific function names
-# (translate, translate_to_en, translate_from_en) the loader will auto-detect them. If the API differs, tell me the function signatures and I'll wire them exactly.
+# Key behaviors:
+# - Loads language.py if present; otherwise attempts to load language.bin (torch.load then pickle).
+# - If the language module exposes translate/translate_to_en/translate_from_en/detect, those are used.
+# - detect_language_safe will consult the language module for detection if available, then fall back to heuristics.
+# - All knowledge retrieval and refinement in /chat is strictly within the resolved topic.
+# - dedupe_sentences preserves sentences as separate lines and avoids turning them into run-ons.
+# - Emoji extraction and a small emoji-sentiment heuristic are used to decide when to append/echo emojis.
+# - Moderation prevents saving toxic memory and prevents adding emojis to responses flagged toxic.
+#
+# Place language.bin and/or language.py and emojis.py in the same folder as this file.
+# Restart the app after placing those files.
 
 from sqlalchemy.pool import NullPool
 import os
@@ -49,9 +53,6 @@ os.environ["SENTENCE_TRANSFORMERS_HOME"] = HF_CACHE_DIR
 # ----- Optional helpers (soft fallbacks) -----
 # Prefer user's emojis.py
 try:
-    # emojis.py is expected to provide at least:
-    # - get_emoji(category: str, intensity: float=0.5) -> str
-    # - get_category_for_mood(mood: str) -> str
     from emojis import get_emoji, get_category_for_mood  # type: ignore
     logger.info("[JusticeAI] Using local emojis.py")
 except Exception:
@@ -85,37 +86,53 @@ except Exception:
     AutoModelForCausalLM = None
     hf_pipeline = None
 
-# ----- Local language model loader (user-provided language.py or language.bin) -----
-language_model = None
+# ----- Local language loader (language.py or language.bin) -----
+language_module = None
 
-def load_local_language_model():
-    global language_model
-    # Try to import a local language module first (language.py with translate API)
+def load_local_language_module():
+    """
+    Attempt to import language.py first. If not present, attempt to load language.bin
+    via torch.load or pickle. The resulting object is stored in `language_module`.
+    The module/object should ideally expose:
+      - translate(text, src, tgt)
+      - translate_to_en(text, src)
+      - translate_from_en(text, tgt)
+      - detect(text) or detect_language(text)
+      - model_info()  (optional)
+    """
+    global language_module
+    # Try language.py module import
     try:
-        import language as local_language_module  # type: ignore
-        language_model = local_language_module
-        logger.info("[JusticeAI] Loaded local language module (language.py)")
+        import language as lm  # type: ignore
+        language_module = lm
+        logger.info("[JusticeAI] Loaded language.py module")
         return
     except Exception:
         pass
-    # If language.bin exists, try to load with torch or pickle
-    try:
-        bin_path = Path("language.bin")
-        if bin_path.exists():
+    # Try language.bin next (torch.load then pickle)
+    bin_path = Path("language.bin")
+    if bin_path.exists():
+        try:
             try:
-                language_model = torch.load(str(bin_path), map_location="cpu")
-                logger.info("[JusticeAI] Loaded local language model from language.bin via torch.load")
-            except Exception:
-                import pickle
-                with open(bin_path, "rb") as f:
-                    language_model = pickle.load(f)
-                logger.info("[JusticeAI] Loaded local language model from language.bin via pickle")
-    except Exception as e:
-        language_model = None
-        logger.warning(f"[JusticeAI] Failed to load local language model: {e}")
+                language_module = torch.load(str(bin_path), map_location="cpu")
+                logger.info("[JusticeAI] Loaded language.bin via torch.load")
+                return
+            except Exception as e:
+                logger.info(f"[JusticeAI] torch.load failed for language.bin: {e}")
+            # fallback to pickle
+            import pickle
+            with open(bin_path, "rb") as f:
+                language_module = pickle.load(f)
+            logger.info("[JusticeAI] Loaded language.bin via pickle")
+            return
+        except Exception as e:
+            language_module = None
+            logger.warning(f"[JusticeAI] Failed to load language.bin: {e}")
+    else:
+        logger.info("[JusticeAI] No language.py or language.bin found in cwd")
 
-# Attempt early load
-load_local_language_model()
+# attempt early load
+load_local_language_module()
 
 # ----- Config (env) -----
 ADMIN_KEY = os.environ.get("ADMIN_KEY")
@@ -288,7 +305,7 @@ class JusticeBrain:
         self.capabilities["embed_available"] = embed_model is not None
         self.capabilities["moderator"] = moderator is not None
         self.capabilities["llm_reflect"] = llm_model is not None and llm_tokenizer is not None
-        self.capabilities["translation"] = (AutoTokenizer is not None and AutoModelForSeq2SeqLM is not None) or (language_model is not None)
+        self.capabilities["translation"] = (AutoTokenizer is not None and AutoModelForSeq2SeqLM is not None) or (language_module is not None)
         self.capabilities["ann"] = False  # FAISS not wired yet (scaffold)
         logger.info(f"[JusticeBrain] Capabilities: {self.capabilities}")
 
@@ -360,17 +377,18 @@ def sanitize_knowledge_text(t: Any) -> str:
         s = s[1:-1].strip()
     return " ".join(s.split())
 
-def dedupe_sentences(text):
+def dedupe_sentences(text: str) -> str:
     """
-    Split text into sentences (respecting newlines) and dedupe while preserving order.
-    Return with sentences separated by newlines (no joining into run-ons).
+    Split text into sentences (respecting existing newlines) and dedupe while preserving order.
+    Return a string where sentences are separated by single newlines (no joining into run-on paragraphs).
     """
     if not text:
         return text
     sentences = []
     seen = set()
-    # Respect explicit newlines first
+    # Respect explicit newlines
     for chunk in re.split(r'\n+', text):
+        # Split on punctuation boundaries but keep them
         parts = re.split(r'(?<=[.?!])\s+', chunk)
         for sent in parts:
             s = sent.strip()
@@ -382,8 +400,7 @@ def dedupe_sentences(text):
             sentences.append(s)
     return "\n".join(sentences)
 
-# Emoji utilities: detect, extract and classify basic sentiment from emojis.
-# This is a conservative heuristic and will be used to inform mood detection and emoji decisions.
+# Emoji detection and heuristics
 _EMOJI_PATTERN = re.compile(
     "["                              
     "\U0001F600-\U0001F64F"  # emoticons
@@ -395,31 +412,21 @@ _EMOJI_PATTERN = re.compile(
     "]+", flags=re.UNICODE
 )
 
-_POS_EMOJI_RANGES = [
-    ("\U0001F600", "\U0001F606"),  # grinning, smiling
-    ("\U0001F60A", "\U0001F60F"),  # smiling variants
-    ("\U0001F642", "\U0001F60D")
-]
-_NEG_EMOJI_RANGES = [
-    ("\U0001F61E", "\U0001F626"),  # sad/concerned faces
-    ("\U0001F62A", "\U0001F626")
-]
-
 def extract_emojis(text: str) -> List[str]:
     if not text:
         return []
     return _EMOJI_PATTERN.findall(text)
 
 def emoji_sentiment_score(emojis: List[str]) -> float:
-    # Returns score in [-1.0, 1.0], positive -> positive sentiment
     if not emojis:
         return 0.0
     score = 0.0
+    total = 0
     for e in "".join(emojis):
         ord_val = ord(e)
-        # quick heuristics: common smiley ranges
-        if 0x1F600 <= ord_val <= 0x1F64F:  # emoticons
-            # smiles ~ positive; frowns closer to 0x1F61E negative
+        total += 1
+        if 0x1F600 <= ord_val <= 0x1F64F:
+            # smiles a bit positive, frowns negative
             if ord_val in range(0x1F600, 0x1F607) or ord_val in range(0x1F60A, 0x1F60F):
                 score += 1.0
             elif ord_val in range(0x1F61E, 0x1F626):
@@ -430,55 +437,112 @@ def emoji_sentiment_score(emojis: List[str]) -> float:
             score += 0.1
         else:
             score += 0.0
-    return max(-1.0, min(1.0, score / max(1, len(emojis))))
+    return max(-1.0, min(1.0, score / max(1, total)))
 
 def detect_language_safe(text: str) -> str:
     """
-    Heuristic language detection only (langdetect removed). Returns an ISO-ish short code or 'und'.
-    Conservative bias toward English for short ASCII phrases.
+    Prefer the local language module detection if available (language.detect or language.detect_language).
+    Then use greeting heuristics and Unicode ranges to detect CJK/JP. Conservative fallback is 'en'.
     """
     text = (text or "").strip()
     if not text:
         return "en"
+    # 1) local language module detection
+    try:
+        global language_module
+        if language_module is not None:
+            # Prefer explicit detect functions if provided
+            if hasattr(language_module, "detect_language"):
+                try:
+                    lang = language_module.detect_language(text)
+                    if lang:
+                        return lang
+                except Exception:
+                    pass
+            if hasattr(language_module, "detect"):
+                try:
+                    lang = language_module.detect(text)
+                    if lang:
+                        return lang
+                except Exception:
+                    pass
+            # Some wrappers expose model_info with detection capability indication
+            if hasattr(language_module, "model_info"):
+                try:
+                    info = language_module.model_info()
+                    # no rigid rule; if model_info exposes a 'detect' attribute we could try it
+                except Exception:
+                    pass
+    except Exception:
+        pass
+
+    # 2) greeting/keyword heuristics
     lower = text.lower()
-    short_ascii = bool(re.fullmatch(r"[\x20-\x7E]+", text))
-    common_en = {"hi", "hello", "hey", "how are you", "ok", "okay", "thanks", "thank you", "yes", "no", "please", "help"}
-    if len(lower.split()) <= 4 and short_ascii:
-        for w in common_en:
-            if w in lower:
-                return "en"
+    greeting_map = {
+        "hola": "es", "gracias": "es", "adios": "es",
+        "bonjour": "fr", "salut": "fr",
+        "hallo": "de", "guten morgen": "de",
+        "ciao": "it", "buongiorno": "it",
+        "olá": "pt", "obrigado": "pt",
+        "привет": "ru", "здравствуйте": "ru",
+        "こんにちは": "ja", "こんばんは": "ja",
+        "你好": "zh", "谢谢": "zh", "안녕하세요": "ko"
+    }
+    for k, v in greeting_map.items():
+        if k in lower:
+            return v
+
+    # 3) Unicode heuristics: Hiragana/Katakana -> Japanese, CJK -> Chinese, Hangul -> Korean
+    if re.search(r'[\u3040-\u30ff]', text):
+        return "ja"
+    if re.search(r'[\u4e00-\u9fff]', text):
+        return "zh"
+    if re.search(r'[\uac00-\ud7af]', text):
+        return "ko"
+
+    # 4) ASCII fallback: if text contains mostly ASCII letters and common english words, treat as 'en'
     letters = re.findall(r'[A-Za-z]', text)
     if len(letters) >= max(1, len(text) / 4):
         return "en"
+
+    # Conservative default
     return "und"
 
 def translate_to_english(text: str, src_lang: str) -> str:
     """
-    Prefer a local language model if available (language.py or language.bin).
-    If not available, fall back to Helsinki-NLP models via transformers.
+    Use the local language module (language_module) if present. Otherwise fall back to Helsinki models.
     """
     if not text:
         return text
     src = (src_lang.split('-')[0].lower() if src_lang else "und")
     if src in ("en", "eng", "", "und"):
         return text
-    # Use local language_model if present
-    if language_model is not None:
-        try:
-            # Common possible APIs: translate_to_en(text, src) or translate(text, src, tgt)
-            if hasattr(language_model, "translate_to_en"):
-                return language_model.translate_to_en(text, src)
-            if hasattr(language_model, "translate"):
+    # prefer language_module
+    try:
+        global language_module
+        if language_module is not None:
+            if hasattr(language_module, "translate_to_en"):
+                try:
+                    return language_module.translate_to_en(text, src)
+                except Exception:
+                    pass
+            if hasattr(language_module, "translate"):
                 try:
-                    return language_model.translate(text, src, "en")
+                    return language_module.translate(text, src, "en")
                 except TypeError:
-                    return language_model.translate(text)
-            if isinstance(language_model, dict):
-                key = (src, "en")
-                return language_model.get(key, text)
-        except Exception as e:
-            logger.warning(f"Local language model translation failed: {e}")
-    # Validate potential model id shape
+                    try:
+                        return language_module.translate(text)
+                    except Exception:
+                        pass
+            # If language_module is an object with callable method
+            if hasattr(language_module, "__call__") and callable(language_module):
+                try:
+                    return language_module(text, src, "en")
+                except Exception:
+                    pass
+    except Exception as e:
+        logger.debug(f"Local language_module translate attempt failed: {e}")
+    # fallback to Helsinki/transformers if available
     if not re.fullmatch(r"[a-z]{2,3}", src):
         return text
     try:
@@ -495,7 +559,7 @@ def translate_to_english(text: str, src_lang: str) -> str:
             model_name = f"Helsinki-NLP/opus-mt-{src}-en"
             tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=TRANSLATION_CACHE_DIR)
             model = AutoModelForSeq2SeqLM.from_pretrained(model_name, cache_dir=TRANSLATION_CACHE_DIR)
-            _translation_model_cache[f"{src}-en"] = (tokenizer, model)
+            _translation_model_cache[cache_key] = (tokenizer, model)
             inputs = tokenizer([text], return_tensors="pt", truncation=True)
             outputs = model.generate(**inputs, max_length=1024)
             return tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
@@ -514,27 +578,37 @@ def translate_to_english(text: str, src_lang: str) -> str:
 
 def translate_from_english(text: str, tgt_lang: str) -> str:
     """
-    Prefer a local language model if available. Else use Helsinki models.
+    Use the local language module if available; otherwise fall back to Helsinki/transformers.
     """
     if not text:
         return text
     tgt = (tgt_lang.split('-')[0].lower() if tgt_lang else "und")
     if tgt in ("en", "eng", "", "und"):
         return text
-    if language_model is not None:
-        try:
-            if hasattr(language_model, "translate_from_en"):
-                return language_model.translate_from_en(text, tgt)
-            if hasattr(language_model, "translate"):
+    try:
+        global language_module
+        if language_module is not None:
+            if hasattr(language_module, "translate_from_en"):
                 try:
-                    return language_model.translate(text, "en", tgt)
+                    return language_module.translate_from_en(text, tgt)
+                except Exception:
+                    pass
+            if hasattr(language_module, "translate"):
+                try:
+                    return language_module.translate(text, "en", tgt)
                 except TypeError:
-                    return language_model.translate(text)
-            if isinstance(language_model, dict):
-                key = ("en", tgt)
-                return language_model.get(key, text)
-        except Exception as e:
-            logger.warning(f"Local language model (en->tgt) translation failed: {e}")
+                    try:
+                        return language_module.translate(text)
+                    except Exception:
+                        pass
+            if hasattr(language_module, "__call__") and callable(language_module):
+                try:
+                    return language_module(text, "en", tgt)
+                except Exception:
+                    pass
+    except Exception as e:
+        logger.debug(f"Local language_module translate_from_en attempt failed: {e}")
+    # fallback to Helsinki/transformers
     if not re.fullmatch(r"[a-z]{2,3}", tgt):
         return text
     try:
@@ -551,7 +625,7 @@ def translate_from_english(text: str, tgt_lang: str) -> str:
             model_name = f"Helsinki-NLP/opus-mt-en-{tgt}"
             tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=TRANSLATION_CACHE_DIR)
             model = AutoModelForSeq2SeqLM.from_pretrained(model_name, cache_dir=TRANSLATION_CACHE_DIR)
-            _translation_model_cache[f"en-{tgt}"] = (tokenizer, model)
+            _translation_model_cache[cache_key] = (tokenizer, model)
             inputs = tokenizer([text], return_tensors="pt", truncation=True)
             outputs = model.generate(**inputs, max_length=1024)
             return tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
@@ -597,7 +671,7 @@ def is_boilerplate_candidate(s: str) -> bool:
 def generate_creative_reply(matches: List[str]) -> str:
     """
     Combine up to three matches into a concise reply.
-    Preserve sentences as separate lines (no joining into run-ons).
+    Preserve sentence lines (no joining into run-ons).
     """
     clean = []
     seen = set()
@@ -606,14 +680,12 @@ def generate_creative_reply(matches: List[str]) -> str:
         if not s or s in seen or is_boilerplate_candidate(s):
             continue
         seen.add(s)
-        # Keep answer sentences separate
         s = dedupe_sentences(s)
         clean.append(s)
     if not clean:
         return "I’m not sure yet."
     if len(clean) == 1:
         return clean[0]
-    # Return as separate lines; preserve sentence boundaries inside each match
     return "\n\n".join(clean[:3])
 
 def infer_topic_from_message(msg: str, known_topics=None) -> str:
@@ -636,8 +708,7 @@ def infer_topic_from_message(msg: str, known_topics=None) -> str:
 
 def refine_or_update(matches, new_text, new_reply, confidence, topic="general"):
     """
-    Decide whether to update existing knowledge or insert a new entry based on similarity.
-    This function ONLY touches rows in the provided topic (enforced).
+    Update or insert knowledge but ONLY inside the given topic.
     """
     try:
         if embed_model is None:
@@ -661,7 +732,6 @@ def refine_or_update(matches, new_text, new_reply, confidence, topic="general"):
         best_score = float(sims[best_idx])
         if best_score > 0.75:
             kid = ids[best_idx]
-            # Check manual flag and prevent contradictory overwrite within topic
             with engine.begin() as conn:
                 row = conn.execute(sql_text("SELECT meta FROM knowledge WHERE id = :id"), {"id": kid}).fetchone()
             is_manual = False
@@ -672,7 +742,6 @@ def refine_or_update(matches, new_text, new_reply, confidence, topic="general"):
             except Exception:
                 is_manual = False
             if is_manual and confidence < 0.85:
-                # Do not overwrite manual high-confidence entries unless very confident; append refined reply note
                 with engine.begin() as conn:
                     conn.execute(
                         sql_text("UPDATE knowledge SET reply = :r, updated_at = CURRENT_TIMESTAMP WHERE id = :id"),
@@ -691,7 +760,6 @@ def refine_or_update(matches, new_text, new_reply, confidence, topic="general"):
                     sql_text("INSERT INTO knowledge (text, reply, language, embedding, category, topic, confidence, meta) VALUES (:t, :r, :lang, :e, 'learned', :topic, :conf, :meta)"),
                     {"t": new_text, "r": new_reply or "", "lang": "en", "e": emb, "topic": topic, "conf": min(0.7, float(confidence)), "meta": json.dumps({"refined": True})}
                 )
-        # bump version on update
         global knowledge_version
         knowledge_version += 1
     except Exception as e:
@@ -699,13 +767,11 @@ def refine_or_update(matches, new_text, new_reply, confidence, topic="general"):
 
 def detect_mood(text: str) -> str:
     """
-    Detect mood from text and emojis. Return 'positive', 'negative', or 'neutral'.
-    This integrates emoji sentiment heuristics.
+    Detect mood using words and emoji heuristics.
     """
     lower = (text or "").lower()
     positive = ["great", "thanks", "awesome", "happy", "love", "excellent", "cool", "yes", "good", "success", "helpful", "useful", "thank you"]
-    negative = ["sad", "bad", "problem", "angry", "hate", "fail", "no", "error", "not working", "disadvantage", "issue", "problem"]
-    # emoji influence
+    negative = ["sad", "bad", "problem", "angry", "hate", "fail", "no", "error", "not working", "disadvantage", "issue"]
     emojis = extract_emojis(text)
     e_score = emoji_sentiment_score(emojis)
     if any(w in lower for w in positive) or e_score > 0.3:
@@ -717,35 +783,25 @@ def detect_mood(text: str) -> str:
 def should_append_emoji(user_text: str, reply_text: str, mood: str, flags: Dict[str, Any]) -> str:
     """
     Decide whether to append/echo an emoji and which one.
-    Rules:
-    - Do not append if moderation flagged toxic.
-    - Do not append if the reply already contains emoji.
-    - Prefer echoing user's emoji if present (amplify or acknowledge).
-    - Only append for short replies and when mood non-neutral or user used emojis.
-    - Use get_emoji/get_category_for_mood (from emojis.py) if available.
-    Returns the emoji string to append or empty string.
+    Conservative rules to avoid inappropriate emoji use.
     """
     if flags.get("toxic"):
         return ""
-    # If reply already has emoji, do not add
+    # If reply already contains emoji, do not add
     if extract_emojis(reply_text):
         return ""
     user_emojis = extract_emojis(user_text)
     if user_emojis:
-        # Echo the first user emoji if it's positive-ish, otherwise map mood
         user_score = emoji_sentiment_score(user_emojis)
         if user_score >= 0.2:
-            # echo a positive emoji
             try:
                 cat = get_category_for_mood("positive")
                 return get_emoji(cat, intensity=min(1.0, 0.5 + user_score))
             except Exception:
                 return user_emojis[0] if user_emojis else ""
         elif user_score <= -0.2:
-            # avoid adding negative emoji; reflect neutrally
             return ""
         else:
-            # for neutral user emoji, optionally add a small positive emoji for short replies
             if len(reply_text) < 200:
                 try:
                     cat = get_category_for_mood("neutral")
@@ -753,13 +809,10 @@ def should_append_emoji(user_text: str, reply_text: str, mood: str, flags: Dict[
                 except Exception:
                     return ""
             return ""
-    # No user emoji: use mood and reply constraints
     if mood == "neutral":
         return ""
-    # do not add emoji for long or formal replies
     if len(reply_text) > 400:
         return ""
-    # avoid adding when reply contains code-like chars
     if re.search(r'[\{\}\[\]\(\)]', reply_text):
         return ""
     try:
@@ -770,9 +823,8 @@ def should_append_emoji(user_text: str, reply_text: str, mood: str, flags: Dict[
 
 def synthesize_final_reply(en_msg: str, matches: List[str], llm_suggestion: str, intent: str, detected_lang: str) -> str:
     """
-    Combine matches and llm_suggestion but JusticeAI always decides final text.
-    - Preserve sentence boundaries (no run-ons).
-    - Return reply in English (later translated if needed).
+    Combine knowledge matches and optional LLM suggestion into a final English reply.
+    Preserve lines, do not join sentences into run-ons.
     """
     pieces = []
     for m in matches:
@@ -785,9 +837,7 @@ def synthesize_final_reply(en_msg: str, matches: List[str], llm_suggestion: str,
                 pieces.append(sent)
     if not pieces:
         return "I don't have enough context — could you add more details or add knowledge with /add?"
-    # Compose using up to 3 pieces; keep them separated by blank lines
     reply = "\n\n".join(pieces[:3])
-    # Intent-specific formatting: solutions -> bullets; others keep lines
     if intent == "solution":
         bullets = []
         for p in re.split(r'\n+', reply):
@@ -806,7 +856,7 @@ def synthesize_final_reply(en_msg: str, matches: List[str], llm_suggestion: str,
 # ----- Startup: load models & background loops -----
 @app.on_event("startup")
 async def startup_event():
-    global embed_model, spell, moderator, llm_tokenizer, llm_model, startup_time
+    global embed_model, spell, moderator, llm_tokenizer, llm_model, startup_time, language_module
     t0 = time.time()
     logger.info("[JusticeAI] Starting component loading...")
 
@@ -875,8 +925,18 @@ async def startup_event():
         model_progress["llm"]["status"] = "error"
         logger.warning(f"[JusticeAI] Could not load local LLM: {e}")
 
-    # reload language model in case startup changed cwd
-    load_local_language_model()
+    # reload language module in case files were placed before startup
+    load_local_language_module()
+    if language_module is not None:
+        try:
+            if hasattr(language_module, "model_info"):
+                info = language_module.model_info()
+                logger.info(f"[JusticeAI] language module info: {info}")
+            else:
+                # attempt a small introspection
+                logger.info(f"[JusticeAI] language module type: {type(language_module)}")
+        except Exception as e:
+            logger.debug(f"[JusticeAI] language module introspect failed: {e}")
 
     startup_time = round(time.time() - t0, 2)
     logger.info(f"[JusticeAI] Startup completed in {startup_time}s")
@@ -903,12 +963,9 @@ async def startup_event():
     def background_learning_loop():
         while True:
             try:
-                # Collect recent user interactions for learning
                 with engine.begin() as conn:
                     mem_rows = conn.execute(sql_text("SELECT text, reply, topic, confidence FROM user_memory ORDER BY created_at DESC LIMIT 200")).fetchall()
                     knowledge_rows = conn.execute(sql_text("SELECT text, reply, topic FROM knowledge ORDER BY created_at DESC LIMIT 200")).fetchall()
-
-                # Use LLM for suggestions on each memory (if available)
                 if llm_model and llm_tokenizer and mem_rows:
                     for mem in mem_rows:
                         user_text = mem[0] or ""
@@ -919,27 +976,29 @@ async def startup_event():
                             inputs = llm_tokenizer(prompt, return_tensors="pt", truncation=True, max_length=512)
                             outputs = llm_model.generate(**inputs, max_length=256, do_sample=True, temperature=0.7)
                             suggestion = llm_tokenizer.decode(outputs[0], skip_special_tokens=True)
-                            # Use suggestion to refine knowledge in a conservative way
                             conf = float(mem[3] or 0)
                             if suggestion and conf >= 0.2:
-                                # refine/update is topic-scoped
                                 refine_or_update([], user_text, suggestion, conf, topic=topic)
                                 logger.debug(f"[Background AGI] Refined knowledge for topic={topic}")
                         except Exception as e:
                             logger.debug(f"[Background AGI] LLM suggestion error for memory: {e}")
-                # Mark learning event
                 record_learn_event()
             except Exception as e:
                 logger.warning(f"[Background AGI] Learning loop error: {e}")
-            time.sleep(60)  # run every minute
-
+            time.sleep(60)
     threading.Thread(target=background_learning_loop, daemon=True).start()
 
 # ----- Endpoints -----
 @app.get("/model-status")
 async def model_status():
     response_progress = {k: dict(v) for k, v in model_progress.items()}
-    return {"model_loaded": embed_model is not None, "model_progress": response_progress, "model_load_times": model_load_times, "startup_time_s": startup_time}
+    lang_info = None
+    try:
+        if language_module is not None and hasattr(language_module, "model_info"):
+            lang_info = language_module.model_info()
+    except Exception:
+        lang_info = {"info": "unavailable"}
+    return {"model_loaded": embed_model is not None, "model_progress": response_progress, "model_load_times": model_load_times, "startup_time_s": startup_time, "language_module": lang_info}
 
 @app.get("/health")
 async def health_check():
@@ -1048,7 +1107,7 @@ async def add_knowledge(data: dict = Body(...)):
         return JSONResponse(status_code=400, content={"error": "Text is required"})
     detected = detect_language_safe(text_data)
     if detected and detected.split("-")[0].lower() not in ("en", "eng", "und"):
-        if AutoTokenizer is not None and AutoModelForSeq2SeqLM is not None or language_model is not None:
+        if AutoTokenizer is not None and AutoModelForSeq2SeqLM is not None or language_module is not None:
             try:
                 text_data = translate_to_english(text_data, detected)
                 detected = "en"
@@ -1081,7 +1140,6 @@ async def add_knowledge(data: dict = Body(...)):
                     sql_text("INSERT INTO knowledge (text, reply, language, category, topic, confidence, meta) VALUES (:t, :r, :lang, 'general', :topic, :conf, :meta)"),
                     {"t": text_data, "r": reply, "lang": "en", "topic": topic, "conf": 0.9, "meta": json.dumps({"manual": True})}
                 )
-        # bump version for caches and indexes
         global knowledge_version
         knowledge_version += 1
         record_learn_event()
@@ -1136,7 +1194,7 @@ async def add_bulk(data: List[dict] = Body(...)):
             errors.append({"index": i, "error": str(e)})
     return {"added": added, "errors": errors}
 
-# Justice Brain Backend — /chat endpoint (multilingual, internal chain reasoning, LLM only for self-improvement)
+# ----- /chat endpoint -----
 @app.post("/chat")
 async def chat(request: Request, data: dict = Body(...)):
     t0 = time.time()
@@ -1146,12 +1204,11 @@ async def chat(request: Request, data: dict = Body(...)):
     user_id = hashlib.sha256(f"{user_ip}-{username}".encode()).hexdigest()
     topic_hint = str(data.get("topic", "") or "").strip()
     detected_lang = detect_language_safe(raw_msg)
-    # Force reply language conservative: if likely English, reply in English
-    likely_en = (detected_lang in ("en", "eng", "") or (len((raw_msg or "").split()) <= 4 and re.fullmatch(r"[\x20-\x7E]+", raw_msg or "") is not None))
-    reply_lang = "en" if (detected_lang in ("und", "") or likely_en) else detected_lang
+    # If detection returns 'und', keep und; otherwise set reply_lang to detected language.
+    reply_lang = detected_lang if detected_lang and detected_lang != "und" else "en"
     user_force_save = bool(data.get("save_memory", False))
 
-    # Spell correction (optional)
+    # Optional spell correction
     if spell is not None:
         try:
             words = raw_msg.split()
@@ -1165,7 +1222,7 @@ async def chat(request: Request, data: dict = Body(...)):
     else:
         msg_corrected = raw_msg
 
-    # Simple intent classifier
+    # Intent classifier
     def classify_intent_local(text: str) -> str:
         t = text.lower()
         if any(k in t for k in ["why", "para qué", "por qué"]):
@@ -1192,7 +1249,7 @@ async def chat(request: Request, data: dict = Body(...)):
     else:
         topic = topic_hint
 
-    # Load knowledge only for the resolved topic (topic-scoped enforcement)
+    # Load knowledge strictly for this topic only
     try:
         with engine.begin() as conn:
             rows = conn.execute(sql_text("SELECT id, text, reply, language, embedding, topic FROM knowledge WHERE topic = :topic ORDER BY created_at DESC"), {"topic": topic}).fetchall()
@@ -1205,12 +1262,12 @@ async def chat(request: Request, data: dict = Body(...)):
     knowledge_langs = [r[3] or "en" for r in rows]
     knowledge_topics = [r[5] or "general" for r in rows]
 
-    # Translate the user message to English if needed
+    # Translate the user message to English if needed (for retrieval/synthesis)
     en_msg = msg_corrected
     if detected_lang and detected_lang.split("-")[0].lower() not in ("en", "eng", "", "und"):
         en_msg = translate_to_english(msg_corrected, detected_lang)
 
-    # Embedding-based retrieval, limited to this topic only
+    # Embedding-based retrieval (topic-scoped)
     matches = []
     confidence = 0.0
     knowledge_embeddings = None
@@ -1242,7 +1299,7 @@ async def chat(request: Request, data: dict = Body(...)):
             matches = [c for _, _, c in filtered]
             confidence = filtered[0][1] if filtered else 0.0
         else:
-            # fallback simple substring matching within topic texts only
+            # fallback: substring search inside topic texts
             for idx, ktext in enumerate(knowledge_texts):
                 ktext_lang = detect_language_safe(ktext)
                 ktext_en = translate_to_english(ktext, ktext_lang) if ktext_lang != "en" else ktext
@@ -1254,7 +1311,7 @@ async def chat(request: Request, data: dict = Body(...)):
         matches = knowledge_replies[:3] if knowledge_replies else []
         confidence = 0.0
 
-    # Internal reasoning scratchpad: build structured notes from matches and message
+    # Build scratchpad and synthesize
     def build_reasoning_scratchpad(question_en: str, facts_en: List[str]) -> Dict[str, Any]:
         scratch = {
             "question": question_en,
@@ -1290,17 +1347,12 @@ async def chat(request: Request, data: dict = Body(...)):
             return "Solutions:\n- " + "\n- ".join(steps[:5])
         if intent_label == "why":
             return base + " It is useful because it provides direct access to relevant information and supports faster decision-making."
-        if intent_label == "advantage":
-            return base
-        if intent_label == "disadvantage":
-            return base
         return base
 
-    # Build scratchpad and synthesize answer (LLM not used for user reply)
     scratchpad = build_reasoning_scratchpad(en_msg, matches)
     reply_en = synthesize_from_scratchpad(scratchpad, intent)
 
-    # Optional: ask local LLM for reflection to improve internal knowledge only (not for direct reply)
+    # Optional LLM reflection for knowledge refinement (not for user reply)
     llm_suggestion = ""
     try:
         if llm_model and llm_tokenizer and matches:
@@ -1318,17 +1370,7 @@ async def chat(request: Request, data: dict = Body(...)):
         logger.debug(f"LLM reflection error: {e}")
         llm_suggestion = ""
 
-    # Decision trace (not returned by default)
-    steps = []
-    if matches and confidence >= 0.6:
-        steps.append(f"Direct match with confidence={confidence:.2f}")
-    elif matches and confidence >= 0.35:
-        steps.append(f"Synthesized from top matches with confidence ~{confidence:.2f}")
-    else:
-        steps.append("Scratchpad synthesis")
-
-    # Compose final reply using Justice Brain's internal synthesis logic
-    reply_en = ""
+    # Compose final reply (knowledge-first, topic-scoped)
     steps = []
     if matches and confidence >= 0.6:
         reply_en = matches[0]
@@ -1341,22 +1383,14 @@ async def chat(request: Request, data: dict = Body(...)):
             if matches or llm_suggestion:
                 reply_en = synthesize_final_reply(en_msg, matches, llm_suggestion, intent, "en")
             else:
-                if embed_model is not None and knowledge_embeddings is not None:
-                    try:
-                        reply_en = synthesize_final_reply(en_msg, [], llm_suggestion, intent, "en")
-                        steps.append("Synthesized from limited items")
-                    except Exception:
-                        reply_en = "I don't have enough context yet — can you give more details?"
-                        steps.append("Fallback related-items failure")
-                else:
-                    reply_en = "I don't have enough context yet — can you give more details?"
-                    steps.append("No embedding model available")
+                reply_en = "I don't have enough context yet — can you give more details?"
+                steps.append("No relevant matches")
         except Exception as e:
             logger.warning(f"Synthesis error: {e}")
             reply_en = "I don't have enough context yet — can you give more details?"
             steps.append("Synthesis fallback")
 
-    # Postprocess for intent (English-only; final translation happens once at the end)
+    # Postprocess for intent
     def postprocess_for_intent_en(reply_text: str, intent_label: str) -> str:
         if intent_label == "why":
             suf = " It is useful because it provides direct access to relevant information and supports faster decision-making."
@@ -1384,12 +1418,9 @@ async def chat(request: Request, data: dict = Body(...)):
     reply_en = postprocess_for_intent_en(reply_en, intent)
     reply_en = dedupe_sentences(reply_en)
 
-    # Mood & emoji handling
-    mood = detect_mood(raw_msg + " " + reply_en)
-    emoji = ""
+    # Moderation check for user message (prevent saving toxic memory)
     flags = {}
     try:
-        # Moderation (prevent toxic content from being saved)
         if moderator is not None:
             mod_result = moderator(raw_msg[:1024])
             if isinstance(mod_result, list) and len(mod_result) > 0:
@@ -1400,11 +1431,12 @@ async def chat(request: Request, data: dict = Body(...)):
     except Exception:
         pass
 
-    # Decide whether to append or echo emoji
+    # Mood & emoji: detect mood from user message and reply, then decide emoji
+    mood = detect_mood(raw_msg + " " + reply_en)
+    emoji = ""
     try:
         chosen_emoji = should_append_emoji(raw_msg, reply_en, mood, flags)
         if chosen_emoji:
-            # Append safely (space separator) and ensure length constraint
             if len(reply_en) + len(chosen_emoji) < 1200:
                 reply_en = reply_en + " " + chosen_emoji
                 emoji = chosen_emoji
@@ -1433,7 +1465,6 @@ async def chat(request: Request, data: dict = Body(...)):
                         "topic": topic,
                     }
                 )
-                # Keep recent / high-confidence per topic
                 conn.execute(
                     sql_text("""
                     DELETE FROM user_memory
@@ -1450,18 +1481,21 @@ async def chat(request: Request, data: dict = Body(...)):
     except Exception as e:
         logger.warning(f"user_memory persist error: {e}")
 
-    # OPTIONAL: include steps for debugging only if requested (default: False)
+    # Translate final reply into user's language if needed (use language_module if available)
+    reply_final = reply_en
+    try:
+        if reply_lang and reply_lang.split("-")[0].lower() not in ("en", "eng", "", "und"):
+            reply_final = translate_from_english(reply_en, reply_lang)
+            reply_final = dedupe_sentences(reply_final)
+    except Exception as e:
+        logger.debug(f"Final translation error: {e}")
+        reply_final = reply_en
+
+    # Optional debug steps
     include_steps = bool(data.get("include_steps", False))
     if include_steps and steps:
         reasoning_text = " | ".join(str(s) for s in steps)
-        reply_en = f"{reply_en}\n\n[Reasoning steps: {reasoning_text}]"
-
-    # Always translate once at the end to the fixed user language target
-    if reply_lang and reply_lang.split("-")[0].lower() not in ("en", "eng"):
-        reply_final = translate_from_english(reply_en, reply_lang)
-        reply_final = dedupe_sentences(reply_final)
-    else:
-        reply_final = reply_en
+        reply_final = f"{reply_final}\n\n[Reasoning steps: {reasoning_text}]"
 
     duration = time.time() - t0
     record_request(duration)