Update conversation_logic.py

conversation_logic.py

@@ -1,217 +1,1031 @@
-# conversation_logic.py
+from __future__ import annotations
 
-from typing import List, Optional
-from models import ChatResponse
-from formatting import format_reply
-from quant_solver import solve_quant, is_quant_question
-from question_fallback_router import get_fallback_pack
+import re
+from typing import Any, Dict, List, Optional, Tuple
 
+from context_parser import detect_intent, intent_to_help_mode
+from formatting import format_explainer_response, format_reply
+from generator_engine import GeneratorEngine
+from models import RetrievedChunk, SolverResult
+from quant_solver import is_quant_question
+from question_classifier import classify_question, normalize_category
+from question_fallback_router import question_fallback_router
+from retrieval_engine import RetrievalEngine
+from solver_router import route_solver
+from explainers.explainer_router import route_explainer
 
-# =========================
-# INPUT TYPE DETECTION
-# =========================
+DIRECT_SOLVE_PATTERNS = [
+    r"\bsolve\b",
+    r"\bwhat is\b",
+    r"\bfind\b",
+    r"\bgive (?:me )?the answer\b",
+    r"\bjust the answer\b",
+    r"\banswer only\b",
+    r"\bcalculate\b",
+]
 
-def _is_topic_query(text: str) -> bool:
+CONTROL_PREFIX_PATTERNS = [
+    r"^\s*solve\s*:\s*",
+    r"^\s*solve\s+",
+    r"^\s*question\s*:\s*",
+    r"^\s*q\s*:\s*",
+    r"^\s*hint\s*:\s*",
+    r"^\s*hint\s*$",
+    r"^\s*next hint\s*:\s*",
+    r"^\s*next hint\s*$",
+    r"^\s*another hint\s*:\s*",
+    r"^\s*another hint\s*$",
+    r"^\s*walkthrough\s*:\s*",
+    r"^\s*walkthrough\s*$",
+    r"^\s*step by step\s*:\s*",
+    r"^\s*step by step\s*$",
+    r"^\s*explain\s*:\s*",
+    r"^\s*explain\s*$",
+    r"^\s*method\s*:\s*",
+    r"^\s*method\s*$",
+    r"^\s*continue\s*$",
+    r"^\s*go on\s*$",
+    r"^\s*next step\s*$",
+]
+
+FOLLOWUP_ONLY_INPUTS = {
+    "hint",
+    "a hint",
+    "give me a hint",
+    "can i have a hint",
+    "next hint",
+    "another hint",
+    "next step",
+    "continue",
+    "go on",
+    "walk me through it",
+    "step by step",
+    "walkthrough",
+    "i'm confused",
+    "im confused",
+    "confused",
+    "explain more",
+    "more explanation",
+    "can you explain that",
+    "help me understand",
+    "help",
+}
+
+
+def _clean_text(text: Optional[str]) -> str:
+    return (text or "").strip()
+
+
+def _safe_get_state(session_state: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    return dict(session_state) if isinstance(session_state, dict) else {}
+
+
+def _extract_question_candidates_from_history_item(item: Dict[str, Any]) -> List[str]:
+    if not isinstance(item, dict):
+        return []
+    candidates: List[str] = []
+    for key in ("question_text", "raw_user_text", "content", "text", "message"):
+        value = item.get(key)
+        if isinstance(value, str) and value.strip():
+            candidates.append(value.strip())
+    meta = item.get("meta")
+    if isinstance(meta, dict):
+        for key in ("question_text", "recovered_question_text"):
+            value = meta.get(key)
+            if isinstance(value, str) and value.strip():
+                candidates.append(value.strip())
+        nested_state = meta.get("session_state")
+        if isinstance(nested_state, dict):
+            value = nested_state.get("question_text")
+            if isinstance(value, str) and value.strip():
+                candidates.append(value.strip())
+    return candidates
+
+
+def _is_followup_hint_only(text: str) -> bool:
     low = (text or "").strip().lower()
+    return low in FOLLOWUP_ONLY_INPUTS
+
+
+def _strip_control_prefix(text: str) -> str:
+    cleaned = (text or "").strip()
+    if not cleaned:
+        return ""
+    previous = None
+    while previous != cleaned:
+        previous = cleaned
+        for pattern in CONTROL_PREFIX_PATTERNS:
+            cleaned = re.sub(pattern, "", cleaned, flags=re.I).strip()
+    return cleaned
+
+
+def _sanitize_question_text(text: str) -> str:
+    raw = (text or "").strip()
+    if not raw:
+        return ""
+    lines = [line.strip() for line in raw.splitlines() if line.strip()]
+    for line in lines:
+        candidate = _strip_control_prefix(line)
+        if candidate and not _is_followup_hint_only(candidate):
+            return candidate
+    return _strip_control_prefix(raw)
+
+
+def _looks_like_question_text(text: str) -> bool:
+    t = (text or "").strip()
+    if not t:
+        return False
+    low = t.lower()
+    return any(
+        [
+            "=" in t,
+            "%" in t,
+            bool(re.search(r"\b\d+\s*:\s*\d+\b", t)),
+            bool(re.search(r"[a-zA-Z]\s*[\+\-\*/=]", t)),
+            any(
+                k in low
+                for k in [
+                    "what is",
+                    "find",
+                    "if ",
+                    "how many",
+                    "probability",
+                    "ratio",
+                    "percent",
+                    "equation",
+                    "integer",
+                    "triangle",
+                    "circle",
+                    "mean",
+                    "median",
+                    "average",
+                    "remainder",
+                    "prime",
+                    "factor",
+                    "divisible",
+                    "area",
+                    "perimeter",
+                    "circumference",
+                ]
+            ),
+        ]
+    )
+
+
+def _is_topic_query(text: str) -> bool:
+    low = _clean_text(text).lower()
+    if not low:
+        return False
 
     exact_patterns = [
         "what topic is this question",
         "what topic is this",
+        "what is the topic of this question",
         "what is the topic",
         "what type of question is this",
+        "what type is this question",
         "what kind of question is this",
+        "what kind is this question",
         "what area is this",
-        "what is this testing",
-        "what skill is this testing",
+        "what concept is this",
         "what concept is this testing",
+        "what skill is this testing",
+        "what is this testing",
         "identify the topic",
+        "identify the concept",
         "identify the type of question",
     ]
-    if any(p in low for p in exact_patterns):
+    if any(phrase in low for phrase in exact_patterns):
         return True
 
     if "topic" in low and "this" in low:
         return True
-
     if "testing" in low and "this" in low:
         return True
-
     if "type" in low and "question" in low:
         return True
-
     if "kind" in low and "question" in low:
         return True
 
     return False
 
 
-def _classify_input_type(text: str) -> str:
-    low = (text or "").lower()
+def _specific_topic_from_question(question_text: str, fallback_topic: str, classified_topic: str) -> str:
+    q = _clean_text(question_text).lower()
+    topic = (fallback_topic or classified_topic or "general").lower()
 
-    # ✅ NEW — FIRST PRIORITY
-    if _is_topic_query(text):
-        return "topic_query"
+    if any(k in q for k in ["variability", "spread", "standard deviation"]):
+        return "variability"
+    if any(k in q for k in ["mean", "average"]):
+        return "mean"
+    if "median" in q:
+        return "median"
+    if "range" in q:
+        return "range"
+    if (
+        "ratio" in q
+        or re.search(r"\b[a-z]\s*/\s*[a-z]\b", q)
+        or re.search(r"\b\d+\s*/\s*\d+\b", q)
+    ):
+        return "ratio"
+    if topic == "data" and any(k in q for k in ["dataset", "table", "chart", "graph"]):
+        return "statistics"
+    return topic
 
-    if any(x in low for x in ["hint", "help"]):
-        return "hint"
 
-    if any(x in low for x in ["how", "walkthrough", "steps"]):
-        return "walkthrough"
+def _build_topic_query_reply(question_text: str, fallback_topic: str, classified_topic: str, category: str) -> str:
+    specific = _specific_topic_from_question(question_text, fallback_topic, classified_topic)
+    cat = (category or "").strip()
+
+    if specific == "variability":
+        return (
+            "- This is a statistics / data insight question about variability (spread).\n"
+            "- The key idea is to compare how spread out each dataset is, not which one has the biggest average.\n"
+            "- A good first move is to compare how far the outer values sit from the middle value in each set."
+        )
+    if specific == "statistics":
+        return (
+            "- This is a statistics / data insight question.\n"
+            "- The key skill is spotting which statistical idea matters most, then comparing the answer choices using that idea."
+        )
+    if specific == "algebra":
+        return (
+            "- This is an algebra question.\n"
+            "- The key skill is rewriting the relationship cleanly, then simplifying the expression the question actually asks for."
+        )
+    if specific == "ratio":
+        return (
+            "- This is a ratio question.\n"
+            "- The key skill is turning the ratio into consistent parts and then building the requested expression from those parts."
+        )
+    if specific == "percent":
+        return (
+            "- This is a percent question.\n"
+            "- The key skill is identifying the correct base quantity before applying the percent relationship."
+        )
+
+    label = specific if specific != "general" else (cat.lower() if cat else "quantitative reasoning")
+    return f"- This looks like a {label} question."
 
-    if any(x in low for x in ["answer", "solve"]):
-        return "answer"
 
+def _classify_input_type(raw_user_text: str) -> str:
+    text = _clean_text(raw_user_text).lower()
+    if not text:
+        return "empty"
+    if _is_topic_query(raw_user_text):
+        return "topic_query"
+    if text in {"hint", "a hint", "give me a hint", "can i have a hint"} or text.startswith("hint:"):
+        return "hint"
+    if text in {"next hint", "another hint", "more hint", "more hints", "next step", "continue", "go on"} or text.startswith("next hint:"):
+        return "next_hint"
+    if any(
+        x in text
+        for x in [
+            "walkthrough",
+            "step by step",
+            "i'm confused",
+            "im confused",
+            "confused",
+            "explain more",
+            "help me understand",
+            "method",
+            "explain",
+        ]
+    ):
+        return "confusion"
+    if text.startswith("solve:") or text.startswith("solve "):
+        return "solve"
+    if _looks_like_question_text(_strip_control_prefix(raw_user_text)):
+        return "question"
     return "other"
 
 
-# =========================
-# MAIN ENTRY
-# =========================
+def _is_followup_input(input_type: str) -> bool:
+    return input_type in {"hint", "next_hint", "confusion"}
 
-def generate_response(
+
+def _history_hint_stage(chat_history: Optional[List[Dict[str, Any]]]) -> int:
+    best = 0
+    for item in chat_history or []:
+        if not isinstance(item, dict):
+            continue
+        try:
+            best = max(best, int(item.get("hint_stage", 0) or 0))
+        except Exception:
+            pass
+        meta = item.get("meta")
+        if isinstance(meta, dict):
+            try:
+                best = max(best, int(meta.get("hint_stage", 0) or 0))
+            except Exception:
+                pass
+            nested_state = meta.get("session_state")
+            if isinstance(nested_state, dict):
+                try:
+                    best = max(best, int(nested_state.get("hint_stage", 0) or 0))
+                except Exception:
+                    pass
+    return min(best, 3)
+
+
+def _recover_question_text(
     raw_user_text: str,
-    tone: float,
-    verbosity: float,
-    transparency: float,
+    question_text: Optional[str],
+    chat_history: Optional[List[Dict[str, Any]]],
+    input_type: str,
+) -> str:
+    explicit = _sanitize_question_text(question_text or "")
+    if explicit:
+        return explicit
+    direct_candidate = _sanitize_question_text(raw_user_text)
+    if direct_candidate and _looks_like_question_text(direct_candidate):
+        return direct_candidate
+    if not _is_followup_input(input_type):
+        return direct_candidate
+    for item in reversed(chat_history or []):
+        for candidate in _extract_question_candidates_from_history_item(item):
+            recovered = _sanitize_question_text(candidate)
+            if recovered and not _is_followup_hint_only(recovered) and _looks_like_question_text(recovered):
+                return recovered
+    return ""
+
+
+def _choose_effective_question_text(
+    raw_user_text: str,
+    question_text: Optional[str],
+    input_type: str,
+    state: Dict[str, Any],
+    chat_history: Optional[List[Dict[str, Any]]],
+) -> Tuple[str, bool]:
+    explicit_question = _sanitize_question_text(question_text or "")
+    stored_question = _sanitize_question_text(state.get("question_text", ""))
+    if _is_followup_input(input_type):
+        if explicit_question and _looks_like_question_text(explicit_question):
+            return explicit_question, False
+        direct_candidate = _sanitize_question_text(raw_user_text)
+        if direct_candidate and _looks_like_question_text(direct_candidate):
+            return direct_candidate, False
+        if stored_question and _looks_like_question_text(stored_question):
+            return stored_question, True
+        recovered = _recover_question_text(raw_user_text, question_text, chat_history, input_type)
+        return recovered, True
+    if explicit_question:
+        return explicit_question, False
+    return _sanitize_question_text(raw_user_text), False
+
+
+def _compute_hint_stage(input_type: str, prior_hint_stage: int, fallback_history_stage: int = 0) -> int:
+    base = max(int(prior_hint_stage or 0), int(fallback_history_stage or 0))
+    if input_type in {"solve", "question"}:
+        return 0
+    if input_type == "hint":
+        return min(max(1, base if base > 0 else 1), 3)
+    if input_type == "next_hint":
+        return min((base if base > 0 else 1) + 1, 3)
+    if input_type == "confusion":
+        return 3
+    return min(base, 3)
+
+
+def _update_session_state(
+    state: Dict[str, Any],
+    *,
     question_text: str,
-    options_text: List[str],
     question_id: Optional[str],
+    hint_stage: int,
+    user_last_input_type: str,
+    built_on_previous_turn: bool,
+    help_mode: str,
+    intent: str,
+    topic: Optional[str],
     category: Optional[str],
-) -> ChatResponse:
+) -> Dict[str, Any]:
+    if question_text:
+        state["question_text"] = question_text
+    if question_id:
+        state["question_id"] = question_id
+    state["hint_stage"] = int(hint_stage or 0)
+    state["user_last_input_type"] = user_last_input_type
+    state["built_on_previous_turn"] = bool(built_on_previous_turn)
+    state["help_mode"] = help_mode
+    state["intent"] = intent
+    state["topic"] = topic
+    state["category"] = category
+    return state
+
+
+def _normalize_classified_topic(topic: Optional[str], category: Optional[str], question_text: str) -> str:
+    t = (topic or "").strip().lower()
+    q = (question_text or "").lower()
+    c = normalize_category(category)
+    has_ratio_form = bool(re.search(r"\b\d+\s*:\s*\d+\b", q))
+    has_algebra_form = (
+        "=" in q
+        or bool(re.search(r"\b[xyzabn]\b", q))
+        or bool(re.search(r"\d+[a-z]\b", q))
+        or bool(re.search(r"\b[a-z]\s*[\+\-\*/=]", q))
+    )
+    if t not in {"general_quant", "general", "unknown", ""}:
+        return t
+    if "%" in q or "percent" in q:
+        return "percent"
+    if "ratio" in q or has_ratio_form:
+        return "ratio"
+    if any(k in q for k in ["probability", "chosen at random", "odds", "chance"]):
+        return "probability"
+    if any(k in q for k in ["divisible", "remainder", "prime", "factor"]):
+        return "number_theory"
+    if any(k in q for k in ["circle", "triangle", "perimeter", "area", "circumference", "rectangle"]):
+        return "geometry"
+    if any(k in q for k in ["mean", "median", "average", "variability", "standard deviation"]):
+        return "statistics" if c == "Quantitative" else "data"
+    if has_algebra_form:
+        return "algebra"
+    if c == "DataInsight":
+        return "data"
+    if c == "Verbal":
+        return "verbal"
+    if c == "Quantitative":
+        return "quant"
+    return "general"
+
+
+def _strip_bullet_prefix(text: str) -> str:
+    return re.sub(r"^\s*[-•]\s*", "", (text or "").strip())
+
+
+def _safe_steps(steps: List[str]) -> List[str]:
+    banned_patterns = [
+        r"\bthe answer is\b",
+        r"\banswer:\b",
+        r"\bthat gives\b",
+        r"\bthis gives\b",
+        r"\btherefore\b",
+        r"\bthus\b",
+        r"\bresult is\b",
+        r"\bfinal answer\b",
+    ]
+    cleaned: List[str] = []
+    for step in steps:
+        s = _strip_bullet_prefix(step)
+        lowered = s.lower()
+        if any(re.search(pattern, lowered) for pattern in banned_patterns):
+            continue
+        if s:
+            cleaned.append(s)
+    deduped: List[str] = []
+    seen = set()
+    for step in cleaned:
+        key = step.lower().strip()
+        if key and key not in seen:
+            seen.add(key)
+            deduped.append(step)
+    return deduped
+
+
+def _safe_meta_list(items: Any) -> List[str]:
+    if not items:
+        return []
+    if isinstance(items, list):
+        return [str(x).strip() for x in items if str(x).strip()]
+    if isinstance(items, tuple):
+        return [str(x).strip() for x in items if str(x).strip()]
+    if isinstance(items, str):
+        text = items.strip()
+        return [text] if text else []
+    return []
+
 
-    result = ChatResponse()
+def _safe_meta_text(value: Any) -> Optional[str]:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
 
-    input_type = _classify_input_type(raw_user_text)
 
-    solver_input = question_text if question_text else raw_user_text
+def _extract_explainer_scaffold(explainer_result: Any) -> Dict[str, Any]:
+    scaffold = getattr(explainer_result, "scaffold", None)
+    if scaffold is None:
+        return {}
+    return {
+        "concept": _safe_meta_text(getattr(scaffold, "concept", None)),
+        "ask": _safe_meta_text(getattr(scaffold, "ask", None)),
+        "givens": _safe_meta_list(getattr(scaffold, "givens", [])),
+        "target": _safe_meta_text(getattr(scaffold, "target", None)),
+        "setup_actions": _safe_meta_list(getattr(scaffold, "setup_actions", [])),
+        "intermediate_steps": _safe_meta_list(getattr(scaffold, "intermediate_steps", [])),
+        "first_move": _safe_meta_text(getattr(scaffold, "first_move", None)),
+        "next_hint": _safe_meta_text(getattr(scaffold, "next_hint", None)),
+        "common_traps": _safe_meta_list(getattr(scaffold, "common_traps", [])),
+        "variables_to_define": _safe_meta_list(getattr(scaffold, "variables_to_define", [])),
+        "equations_to_form": _safe_meta_list(getattr(scaffold, "equations_to_form", [])),
+        "answer_hidden": bool(getattr(scaffold, "answer_hidden", True)),
+        "solution_path_type": _safe_meta_text(getattr(scaffold, "solution_path_type", None)),
+        "key_operations": _safe_meta_list(getattr(scaffold, "key_operations", [])),
+        "hint_ladder": _safe_meta_list(getattr(scaffold, "hint_ladder", [])),
+    }
+
+
+def _get_result_steps(result: Optional[SolverResult]) -> List[str]:
+    if result is None:
+        return []
+    display_steps = getattr(result, "display_steps", None)
+    if isinstance(display_steps, list) and display_steps:
+        return _safe_steps(display_steps)
+    result_steps = getattr(result, "steps", None)
+    if isinstance(result_steps, list) and result_steps:
+        return _safe_steps(result_steps)
+    meta = getattr(result, "meta", {}) or {}
+    meta_display_steps = meta.get("display_steps")
+    if isinstance(meta_display_steps, list) and meta_display_steps:
+        return _safe_steps(meta_display_steps)
+    meta_steps = meta.get("steps")
+    if isinstance(meta_steps, list) and meta_steps:
+        return _safe_steps(meta_steps)
+    return []
+
+
+def _apply_safe_step_sanitization(result: Optional[SolverResult]) -> None:
+    if result is None:
+        return
+    safe_steps = _get_result_steps(result)
+    result.steps = list(safe_steps)
+    setattr(result, "display_steps", list(safe_steps))
+    result.meta = result.meta or {}
+    result.meta["steps"] = list(safe_steps)
+    result.meta["display_steps"] = list(safe_steps)
+
+
+def _solver_has_useful_steps(result: Optional[SolverResult]) -> bool:
+    return bool(result is not None and _get_result_steps(result))
+
 
-    is_quant = is_quant_question(solver_input)
+def _answer_path_from_steps(steps: List[str], verbosity: float) -> str:
+    safe_steps = _safe_steps(steps)
+    if not safe_steps:
+        return ""
+    shown_steps = safe_steps[:2] if verbosity < 0.35 else safe_steps[:3] if verbosity < 0.8 else safe_steps
+    return "\n".join(f"- {step}" for step in shown_steps)
 
-    fallback_pack = get_fallback_pack(
-        question_text=solver_input,
+
+def _build_fallback_reply(
+    *,
+    question_id: Optional[str],
+    question_text: str,
+    options_text: Optional[List[str]],
+    topic: Optional[str],
+    category: Optional[str],
+    help_mode: str,
+    hint_stage: int,
+    verbosity: float,
+) -> Tuple[str, Dict[str, Any]]:
+    payload = question_fallback_router.build_response(
+        question_id=question_id,
+        question_text=question_text,
+        options_text=options_text,
+        topic=topic,
         category=category,
+        help_mode=help_mode,
+        hint_stage=hint_stage,
+        verbosity=verbosity,
     )
+    lines = payload.get("lines") or ["Start by identifying the main relationship in the problem."]
+    pack = payload.get("pack") or {}
+    return "\n".join(f"- {line}" for line in lines if str(line).strip()), pack
 
-    question_topic = fallback_pack.get("topic") if fallback_pack else "general"
 
-    # =========================
-    # ✅ TOPIC QUERY HANDLER
-    # =========================
+def _is_direct_solve_request(text: str, intent: str) -> bool:
+    if intent == "answer":
+        return True
+    t = re.sub(r"\s+", " ", (text or "").strip().lower())
+    if any(re.search(p, t) for p in DIRECT_SOLVE_PATTERNS):
+        if not any(word in t for word in ["how", "explain", "why", "method", "hint", "define", "definition", "step"]):
+            return True
+    return False
 
-    if input_type == "topic_query":
 
-        final_topic = question_topic or "general"
-        qlow = (solver_input or "").lower()
+def _is_help_first_mode(help_mode: str) -> bool:
+    return help_mode in {"hint", "walkthrough", "explain", "instruction", "step_by_step"}
 
-        if "variability" in qlow or "spread" in qlow:
-            core = (
-                "This is a statistics / data insight question about variability (spread).\n"
-                "The key skill is comparing how spread out the values are."
-            )
 
-        elif final_topic == "algebra":
-            core = (
-                "This is an algebra question.\n"
-                "It tests how to use a relationship to rewrite or simplify an expression."
-            )
+def _should_try_solver(is_quant: bool, help_mode: str, solver_input: str) -> bool:
+    if not is_quant or not solver_input:
+        return False
+    return help_mode in {"answer", "walkthrough", "instruction", "hint", "step_by_step"}
 
-        elif final_topic == "ratio":
-            core = (
-                "This is a ratio question.\n"
-                "It tests how to turn ratios into consistent parts and use them in expressions."
-            )
 
-        elif final_topic == "percent":
-            core = (
-                "This is a percent question.\n"
-                "It tests identifying the correct base and applying the percent relationship."
-            )
+def _should_prefer_question_support(help_mode: str, fallback_pack: Dict[str, Any]) -> bool:
+    if not fallback_pack:
+        return False
+    support_source = str(fallback_pack.get("support_source", "")).strip().lower()
+    has_specific_content = support_source in {"question_bank", "question_id", "question_text"}
+    if help_mode in {"hint", "walkthrough", "instruction", "step_by_step", "explain"}:
+        return has_specific_content or bool(fallback_pack)
+    return False
 
-        elif final_topic == "statistics":
-            core = (
-                "This is a statistics / data insight question.\n"
-                "It tests identifying the correct statistical concept and comparing datasets."
-            )
 
-        else:
-            core = f"This looks like a {final_topic} question."
-
-        reply = format_reply(
-            core,
-            tone=tone,
-            verbosity=verbosity,
-            transparency=transparency,
-            help_mode="answer",
-            hint_stage=0,
-            topic=final_topic,
+def _minimal_generic_reply(category: Optional[str]) -> str:
+    c = normalize_category(category)
+    if c == "Verbal":
+        return "I can help analyse the wording or logic, but I need the full question text to guide you properly."
+    if c == "DataInsight":
+        return "I can help reason through the data, but I need the full question or chart details to guide you properly."
+    return "Start by identifying the main relationship in the problem."
+
+
+class ConversationEngine:
+    def __init__(
+        self,
+        retriever: Optional[RetrievalEngine] = None,
+        generator: Optional[GeneratorEngine] = None,
+        **kwargs,
+    ) -> None:
+        self.retriever = retriever
+        self.generator = generator
+
+    def generate_response(
+        self,
+        raw_user_text: Optional[str] = None,
+        tone: float = 0.5,
+        verbosity: float = 0.5,
+        transparency: float = 0.5,
+        intent: Optional[str] = None,
+        help_mode: Optional[str] = None,
+        retrieval_context: Optional[List[RetrievedChunk]] = None,
+        chat_history: Optional[List[Dict[str, Any]]] = None,
+        question_text: Optional[str] = None,
+        options_text: Optional[List[str]] = None,
+        question_id: Optional[str] = None,
+        session_state: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ) -> SolverResult:
+        user_text = _clean_text(raw_user_text)
+        state = _safe_get_state(session_state)
+        input_type = _classify_input_type(user_text)
+
+        effective_question_text, built_on_previous_turn = _choose_effective_question_text(
+            raw_user_text=user_text,
+            question_text=question_text,
+            input_type=input_type,
+            state=state,
+            chat_history=chat_history,
         )
+        if _is_followup_input(input_type):
+            built_on_previous_turn = True
 
-        result.reply = reply
-        result.topic = final_topic
-
-        result.meta = {
-            "intent": "topic_query",
-            "response_source": "topic_classifier",
-            "topic": final_topic,
-            "question_text": solver_input,
-            "options_count": len(options_text or []),
-            "category": category,
-            "used_retrieval": False,
-            "used_generator": False,
-        }
+        solver_input = _sanitize_question_text(effective_question_text)
+        question_id = question_id or state.get("question_id")
 
-        return result
+        category = normalize_category(kwargs.get("category"))
+        classification = classify_question(question_text=solver_input, category=category)
+        inferred_category = normalize_category(classification.get("category") or category)
+        question_topic = _normalize_classified_topic(classification.get("topic"), inferred_category, solver_input)
 
-    # =========================
-    # NORMAL FLOW
-    # =========================
+        resolved_intent = intent or detect_intent(user_text, help_mode)
+        if input_type in {"hint", "next_hint"}:
+            resolved_intent = "hint"
+        elif input_type == "confusion":
+            resolved_intent = "walkthrough"
+        elif input_type in {"solve", "question"} and resolved_intent in {"hint", "walkthrough", "step_by_step"}:
+            resolved_intent = "answer"
 
-    solver_result = None
+        resolved_help_mode = help_mode or intent_to_help_mode(resolved_intent)
+        if input_type in {"hint", "next_hint"}:
+            resolved_help_mode = "hint"
+        elif input_type == "confusion":
+            resolved_help_mode = "walkthrough"
+        elif resolved_help_mode == "step_by_step":
+            resolved_help_mode = "walkthrough"
 
-    if is_quant:
-        solver_result = solve_quant(solver_input, options_text)
+        prior_hint_stage = int(state.get("hint_stage", 0) or 0)
+        history_hint_stage = _history_hint_stage(chat_history)
+        hint_stage = _compute_hint_stage(input_type, prior_hint_stage, history_hint_stage)
 
-    # =========================
-    # FALLBACK / SOLVER
-    # =========================
+        is_quant = bool(solver_input) and (
+            inferred_category == "Quantitative" or is_quant_question(solver_input)
+        )
 
-    if solver_result and solver_result.steps:
-        reply_core = "\n".join(f"- {s}" for s in solver_result.steps)
-        source = "solver_steps"
-        topic = solver_result.topic or question_topic
+        result = SolverResult(
+            domain="quant" if is_quant else "general",
+            solved=False,
+            help_mode=resolved_help_mode,
+            topic=question_topic if is_quant else "general",
+            used_retrieval=False,
+            used_generator=False,
+            steps=[],
+            teaching_chunks=[],
+            meta={},
+        )
 
-    elif fallback_pack:
-        reply_core = "\n".join(f"- {s}" for s in fallback_pack.get("answer_path", []))
-        source = "fallback"
-        topic = question_topic
+        solver_result: Optional[SolverResult] = None
+        if _should_try_solver(is_quant, resolved_help_mode, solver_input):
+            try:
+                solver_result = route_solver(solver_input)
+            except Exception:
+                solver_result = None
+            _apply_safe_step_sanitization(solver_result)
 
-    else:
-        reply_core = "Try breaking the problem into smaller steps."
-        source = "generic"
-        topic = "general"
+        explainer_result = None
+        explainer_understood = False
+        explainer_scaffold: Dict[str, Any] = {}
+        if solver_input:
+            try:
+                explainer_result = route_explainer(solver_input)
+            except Exception:
+                explainer_result = None
+        if explainer_result is not None and getattr(explainer_result, "understood", False):
+            explainer_understood = True
+            explainer_scaffold = _extract_explainer_scaffold(explainer_result)
 
-    reply = format_reply(
-        reply_core,
-        tone=tone,
-        verbosity=verbosity,
-        transparency=transparency,
-        help_mode="answer",
-        hint_stage=0,
-        topic=topic,
-    )
+        fallback_reply_core = ""
+        fallback_pack: Dict[str, Any] = {}
+        if solver_input:
+            fallback_reply_core, fallback_pack = _build_fallback_reply(
+                question_id=question_id,
+                question_text=solver_input,
+                options_text=options_text,
+                topic=question_topic,
+                category=inferred_category,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+                verbosity=verbosity,
+            )
 
-    result.reply = reply
-    result.topic = topic
-
-    result.meta = {
-        "intent": "answer",
-        "response_source": source,
-        "topic": topic,
-        "question_text": solver_input,
-        "options_count": len(options_text or []),
-        "category": category,
-        "used_retrieval": False,
-        "used_generator": False,
-    }
+        # Merge solver result into base result only if it agrees with the classified topic.
+        if solver_result is not None:
+            result.meta = result.meta or {}
+            solver_topic = getattr(solver_result, "topic", None) or "unknown"
+            
+            compatible_topics = {
+                question_topic,
+                "general_quant",
+                "general",
+                "unknown",
+            }
+            
+            # allow a few sensible cross-matches
+            if question_topic == "algebra":
+                compatible_topics.update({"ratio"})
+            elif question_topic == "ratio":
+                compatible_topics.update({"algebra"})
+            elif question_topic == "percent":
+                compatible_topics.update({"ratio", "algebra"})
+            
+            if solver_topic in compatible_topics:
+                result = solver_result
+                result.domain = "quant"
+                result.meta = result.meta or {}
+                result.topic = question_topic if question_topic else solver_topic
+                result.meta["solver_topic_accepted"] = solver_topic
+            else:
+                result.meta["solver_topic_rejected"] = solver_topic
+                result.meta["solver_topic_expected"] = question_topic
+                result.topic = question_topic if is_quant else result.topic
+        else:
+            result.meta = result.meta or {}
+            result.topic = question_topic if is_quant else result.topic
+
+        _apply_safe_step_sanitization(result)
+        solver_steps = _get_result_steps(result)
+        solver_has_steps = bool(solver_steps)
+        prefer_question_support = _should_prefer_question_support(resolved_help_mode, fallback_pack)
+        direct_solve_request = _is_direct_solve_request(user_text or solver_input, resolved_intent)
+        solver_topic_ok = result.meta.get("solver_topic_rejected") is None
+
+        result.help_mode = resolved_help_mode
+        result.meta = result.meta or {}
+        result.meta["hint_stage"] = hint_stage
+        result.meta["max_stage"] = 3
+        result.meta["recovered_question_text"] = solver_input
+        result.meta["question_id"] = question_id
+        result.meta["classified_topic"] = question_topic if question_topic else "general"
+        result.meta["explainer_understood"] = explainer_understood
+        result.meta["explainer_scaffold"] = explainer_scaffold
+
+        if input_type == "topic_query":
+            support_topic = fallback_pack.get("topic") if fallback_pack else ""
+            topic_reply_core = _build_topic_query_reply(
+                question_text=solver_input,
+                fallback_topic=support_topic,
+                classified_topic=question_topic if question_topic else "general",
+                category=inferred_category if inferred_category else "General",
+            )
+            reply = format_reply(
+                topic_reply_core,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode="answer",
+                hint_stage=hint_stage,
+                topic=support_topic or question_topic or result.topic,
+            )
+            result.topic = support_topic or question_topic or result.topic
+            result.reply = reply
+            result.help_mode = "answer"
+            result.meta["response_source"] = "topic_classifier"
+            result.meta["question_support_used"] = bool(fallback_pack)
+            result.meta["question_support_source"] = fallback_pack.get("support_source") if fallback_pack else None
+            result.meta["question_support_topic"] = support_topic or None
+            result.meta["help_mode"] = "answer"
+            result.meta["intent"] = "topic_query"
+            result.meta["question_text"] = solver_input or ""
+            result.meta["options_count"] = len(options_text or [])
+            result.meta["category"] = inferred_category if inferred_category else "General"
+            result.meta["user_last_input_type"] = input_type
+            result.meta["built_on_previous_turn"] = built_on_previous_turn
+            state = _update_session_state(
+                state,
+                question_text=solver_input,
+                question_id=question_id,
+                hint_stage=hint_stage,
+                user_last_input_type=input_type,
+                built_on_previous_turn=built_on_previous_turn,
+                help_mode="answer",
+                intent="topic_query",
+                topic=result.topic,
+                category=inferred_category,
+            )
+            result.meta["session_state"] = state
+            result.meta["used_retrieval"] = False
+            result.meta["used_generator"] = False
+            result.meta["can_reveal_answer"] = False
+            result.answer_letter = None
+            result.answer_value = None
+            result.internal_answer = None
+            result.meta["internal_answer"] = None
+            return result
+
+        if fallback_pack and fallback_pack.get("topic") == "statistics":
+            qlow = (solver_input or "").lower()
+            wants_topic = input_type == "topic_query"
+            if any(k in qlow for k in ["variability", "spread", "standard deviation"]):
+                if wants_topic:
+                    fallback_reply_core = (
+                        "- This is a statistics / data insight question about variability (spread).\n"
+                        "- Focus on how spread out each dataset is rather than the average.\n"
+                        "- Compare how far the outer values sit from the middle value in each set."
+                    )
+                elif resolved_help_mode == "answer":
+                    fallback_reply_core = (
+                        "- Notice this is asking about variability, which means spread, not the mean.\n"
+                        "- Compare how far the smallest and largest values sit from the middle value in each dataset.\n"
+                        "- The set with the widest spread has the greatest variability."
+                    )
 
-    return result
+        # Source selection priority:
+        # 1) question-specific fallback for help modes
+        # 2) explainer for explain when specific support is unavailable
+        # 3) solver steps for direct answer / walkthrough fallback
+        # 4) generic fallback
+        if resolved_help_mode == "explain" and prefer_question_support and fallback_reply_core:
+            reply_core = fallback_reply_core
+            result.meta["response_source"] = "question_support"
+            result.meta["question_support_used"] = True
+            result.meta["question_support_source"] = fallback_pack.get("support_source")
+            result.meta["question_support_topic"] = fallback_pack.get("topic")
+            reply = format_reply(
+                reply_core,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+                topic=result.topic,
+            )
+        elif resolved_help_mode == "explain" and explainer_understood:
+            reply = format_explainer_response(
+                result=explainer_result,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+            )
+            result.meta["response_source"] = "explainer"
+            result.meta["explainer_used"] = True
+            result.meta["question_support_used"] = False
+        elif _is_help_first_mode(resolved_help_mode) and prefer_question_support and fallback_reply_core:
+            reply_core = fallback_reply_core
+            result.meta["response_source"] = "question_support"
+            result.meta["question_support_used"] = True
+            result.meta["question_support_source"] = fallback_pack.get("support_source")
+            result.meta["question_support_topic"] = fallback_pack.get("topic")
+            reply = format_reply(
+                reply_core,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+                topic=result.topic,
+            )
+        elif (
+            resolved_help_mode == "answer"
+            and solver_has_steps
+            and result.meta.get("solver_topic_rejected") is None
+            and direct_solve_request
+        ):
+            reply_core = _answer_path_from_steps(solver_steps, verbosity=verbosity)
+            result.meta["response_source"] = "solver_steps"
+            result.meta["question_support_used"] = False
+            reply = format_reply(
+                reply_core,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+                topic=result.topic,
+            )
+        elif (
+            resolved_help_mode == "walkthrough"
+            and solver_has_steps
+            and not prefer_question_support
+            and result.meta.get("solver_topic_rejected") is None
+        ):
+            reply_core = _answer_path_from_steps(solver_steps, verbosity=verbosity)
+            result.meta["response_source"] = "solver_steps"
+            result.meta["question_support_used"] = False
+            reply = format_reply(
+                reply_core,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+                topic=result.topic,
+            )
+        elif fallback_reply_core:
+            reply_core = fallback_reply_core
+            result.meta["response_source"] = "fallback"
+            result.meta["question_support_used"] = bool(fallback_pack)
+            result.meta["question_support_source"] = fallback_pack.get("support_source")
+            result.meta["question_support_topic"] = fallback_pack.get("topic")
+            reply = format_reply(
+                reply_core,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+                topic=result.topic,
+            )
+        else:
+            reply_core = _minimal_generic_reply(inferred_category)
+            if not reply_core.startswith("- "):
+                reply_core = f"- {reply_core}"
+            result.meta["response_source"] = "generic"
+            result.meta["question_support_used"] = False
+            reply = format_reply(
+                reply_core,
+                tone=tone,
+                verbosity=verbosity,
+                transparency=transparency,
+                help_mode=resolved_help_mode,
+                hint_stage=hint_stage,
+                topic=result.topic,
+            )
+
+        # Never reveal final answers during tutoring/help modes.
+        if resolved_help_mode in {"hint", "walkthrough", "explain", "instruction", "step_by_step"}:
+            result.solved = False
+            result.answer_letter = None
+            result.answer_value = None
+            result.internal_answer = None
+            result.meta["internal_answer"] = None
+
+        # Only allow answer metadata to survive for true direct solve requests.
+        can_reveal_answer = bool(result.solved and direct_solve_request and not _is_help_first_mode(resolved_help_mode))
+        result.meta["can_reveal_answer"] = can_reveal_answer
+        if not can_reveal_answer:
+            result.answer_letter = None
+            result.answer_value = None
+            result.internal_answer = None
+            result.meta["internal_answer"] = None
+
+        state = _update_session_state(
+            state,
+            question_text=solver_input,
+            question_id=question_id,
+            hint_stage=hint_stage,
+            user_last_input_type=input_type,
+            built_on_previous_turn=built_on_previous_turn,
+            help_mode=resolved_help_mode,
+            intent=resolved_intent,
+            topic=result.topic,
+            category=inferred_category,
+        )
+
+        result.reply = reply
+        result.help_mode = resolved_help_mode
+        result.meta["help_mode"] = resolved_help_mode
+        result.meta["intent"] = resolved_intent
+        result.meta["question_text"] = solver_input or ""
+        result.meta["options_count"] = len(options_text or [])
+        result.meta["category"] = inferred_category if inferred_category else "General"
+        result.meta["user_last_input_type"] = input_type
+        result.meta["built_on_previous_turn"] = built_on_previous_turn
+        result.meta["session_state"] = state
+        result.meta["used_retrieval"] = False
+        result.meta["used_generator"] = False
+        return result