Upload app.py

app.py

@@ -3,6 +3,14 @@ app.py  –  Article Q&A chatbot
 Runs on:
   • Hugging Face Spaces  (CPU-only, default)
   • Local PC             (CPU or CUDA GPU)
+
+Environment variables
+---------------------
+HF_TOKEN            HuggingFace token for private model repo  (required on HF Space)
+LOCAL_MODE          Set to "1" to force local-PC behaviour     (optional; auto-detected via SPACE_ID)
+LOCAL_MODEL_PATH    Absolute path to the .gguf file on disk    (optional; skips HF hub download)
+GITHUB_TOKEN        GitHub PAT for higher rate-limits          (optional; works without it)
+N_THREADS           Override CPU thread count                  (optional)
 """
 
 import gradio as gr
@@ -15,18 +23,20 @@ from datetime import datetime, timedelta
 from langchain_huggingface import HuggingFaceEmbeddings
 
 # ====================== ENVIRONMENT DETECTION ======================
+# HuggingFace Spaces always set SPACE_ID; absent → we're running locally.
 IS_HF_SPACE = bool(os.environ.get("SPACE_ID"))
 IS_LOCAL    = (not IS_HF_SPACE) or (os.environ.get("LOCAL_MODE", "0") == "1")
 
 def _detect_cuda() -> bool:
     """Return True only when a CUDA device is actually usable by llama-cpp."""
     if not IS_LOCAL:
-        return False
+        return False          # HF free tier is CPU-only
     try:
         import torch
         return torch.cuda.is_available()
     except ImportError:
         pass
+    # Fallback: check for libcuda without torch
     try:
         import ctypes
         ctypes.cdll.LoadLibrary("libcuda.so.1")
@@ -35,7 +45,9 @@ def _detect_cuda() -> bool:
         return False
 
 CUDA_AVAILABLE = _detect_cuda()
+# -1  → offload every layer to GPU;  0 → pure CPU
 N_GPU_LAYERS   = -1 if CUDA_AVAILABLE else 0
+# Use all available cores locally; HF free tier: keep at 2 to avoid OOM
 N_THREADS      = int(os.environ.get("N_THREADS", os.cpu_count() if IS_LOCAL else 2))
 
 # ====================== CONFIG ======================
@@ -58,12 +70,13 @@ GH_NEWS_PATH       = "MorningNewsAgentTest"
 GH_API_ROOT        = "https://api.github.com"
 GH_RAW_ROOT        = "https://raw.githubusercontent.com"
 NEWS_ACCEPTED_EXT  = (".txt", ".md", ".json")
-NEWS_MAX_CHARS_FILE = 2000
-NEWS_MAX_CHARS_TOTAL = 3500
+NEWS_MAX_CHARS_FILE = 2000   # chars kept per file
+NEWS_MAX_CHARS_TOTAL = 3500  # total chars injected into prompt
 NEWS_CACHE_TTL     = timedelta(hours=1)
 
-# Context budget
-CONTEXT_BUDGET_CHARS = 2900
+# Web search
+WEB_MAX_RESULTS    = 5
+WEB_MAX_CHARS      = 2500    # total chars from web injected into prompt
 
 # ====================== LOAD METADATA ======================
 def load_articles_list() -> list[str]:
@@ -135,9 +148,15 @@ def get_vectorstore(backend_name: str):
 _news_cache: dict = {"content": None, "fetched_at": None}
 
 def fetch_morning_news() -> str:
+    """
+    Fetch text/md/json files from the MorningNewsAgentTest directory on GitHub.
+    Results are cached for NEWS_CACHE_TTL to avoid hammering the API.
+    Works with or without a GITHUB_TOKEN (unauthenticated rate-limit: 60 req/hr).
+    """
     global _news_cache
     now = datetime.utcnow()
 
+    # Serve from cache if still fresh
     if _news_cache["content"] is not None and _news_cache["fetched_at"]:
         if now - _news_cache["fetched_at"] < NEWS_CACHE_TTL:
             print("[MorningNews] Serving from cache")
@@ -149,13 +168,16 @@ def fetch_morning_news() -> str:
         headers["Authorization"] = f"token {gh_token}"
 
     try:
+        # List files in the directory
         dir_url = f"{GH_API_ROOT}/repos/{GH_OWNER}/{GH_REPO}/contents/{GH_NEWS_PATH}"
         resp = requests.get(dir_url, headers=headers, timeout=10)
         resp.raise_for_status()
         entries = resp.json()
 
+        # Sort by name descending so the most recent file (date-prefixed) comes first
         entries = sorted(
-            [e for e in entries if e["type"] == "file" and e["name"].lower().endswith(NEWS_ACCEPTED_EXT)],
+            [e for e in entries if e["type"] == "file"
+             and e["name"].lower().endswith(NEWS_ACCEPTED_EXT)],
             key=lambda e: e["name"],
             reverse=True,
         )
@@ -181,9 +203,11 @@ def fetch_morning_news() -> str:
 
     except Exception as e:
         print(f"[MorningNews] Directory listing failed: {e}")
+        # Return stale cache rather than nothing if available
         return _news_cache.get("content") or ""
 
 # ====================== SYSTEM PROMPTS ======================
+# Base prompt – articles only
 SYSTEM_PROMPT_BASE = """You are the reference expert for the articles contained in the training of this model, \
 all extracted from the website robertolofaro.com, and all focused on change.
 # Your Mission
@@ -194,22 +218,34 @@ Do not provide general advice from outside these sources.
 2. Guidelines & Hints: A markdown list of specific answers/guidelines/hints found in the source material.
 """
 
+# Extended prompt – when extra sources are active
 SYSTEM_PROMPT_EXTENDED = """You are the reference expert for the articles contained in the training of this model, \
 all extracted from the website robertolofaro.com, and all focused on change. \
-You have also been provided with supplementary external context (morning news).
+You have also been provided with supplementary external context (morning news results).
 # Your Mission
 Provide a structured response that integrates all available information. \
-Clearly tag each insight with its source label:
+Clearly tag each insight with its source label so the reader can judge its provenance:
   [Articles]     – insight from the trained article corpus
   [MorningNews]  – insight from the morning news briefing
 # Response Format
 1. Executive Summary: A 2-3 sentence overview answering the core query.
 2. Guidelines & Hints: A markdown list of tagged insights from the source material.
-3. Additional Context (when MorningNews is present): brief synthesis of external findings relevant to the query.
+3. Additional Context (when MorningNews are present): \
+   brief synthesis of external findings relevant to the query.
 """
 
 # ====================== CONTEXT BUDGET HELPER ======================
+# Rough token estimate: 1 token ≈ 4 chars for English text.
+# n_ctx=4096 → reserve ~800 for answer, ~400 for system+history → ~2900 chars for context.
+CONTEXT_BUDGET_CHARS = 2900
+
 def _trim_to_budget(parts: list[tuple[str, str]]) -> str:
+    """
+    parts = [(label, text), ...]
+    Allocates the context budget proportionally across available sources,
+    then returns a single assembled context string.
+    """
+    # First pass: measure totals
     totals = [(label, text) for label, text in parts if text.strip()]
     if not totals:
         return ""
@@ -224,10 +260,146 @@ def _trim_to_budget(parts: list[tuple[str, str]]) -> str:
 def generate_response(
     message, history,
     rag_mode, article_filter,
-    use_morning_news,
+    use_morning_news, 
     max_tokens, temperature, top_p, repeat_penalty,
 ):
     has_extra = use_morning_news
     system_prompt = SYSTEM_PROMPT_EXTENDED if has_extra else SYSTEM_PROMPT_BASE
 
-    full_prompt = f"<|im_start|>system\n{system_prompt}<|
+    full_prompt = f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
+
+    # Keep the last 4 turns to limit context pressure
+    for msg in history[-4:]:
+        full_prompt += f"<|im_start|>{msg['role']}\n{msg['content']}<|im_end|>\n"
+
+    # ---- Gather context from all active sources ----
+    context_parts: list[tuple[str, str]] = []
+
+    # 1. RAG (vectorstore)
+    backend = BACKENDS.get(rag_mode)
+    if backend:
+        vs = get_vectorstore(backend)
+        if vs:
+            try:
+                filt = {"article_category": article_filter} if article_filter != "All categories" else None
+                docs = vs.similarity_search(message, k=5, filter=filt)
+                rag_text = "\n\n".join(
+                    f"[Cat: {d.metadata.get('article_category','N/A')}] {d.page_content[:700]}"
+                    for d in docs
+                )
+                context_parts.append(("ARTICLES CONTEXT", rag_text))
+            except Exception as e:
+                print(f"[RAG] similarity_search failed: {e}")
+
+    # 2. Morning News
+    if use_morning_news:
+        news = fetch_morning_news()
+        if news:
+            context_parts.append(("MORNING NEWS BRIEFING", news))
+
+    # ---- Assemble context within token budget ----
+    context = _trim_to_budget(context_parts)
+
+    if context:
+        full_prompt += f"<|im_start|>user\nContext:\n{context}\n\nQuestion: {message}<|im_end|>\n"
+    else:
+        full_prompt += f"<|im_start|>user\n{message}<|im_end|>\n"
+
+    full_prompt += "<|im_start|>assistant\n"
+
+    # ---- Inference parameters ----
+    max_tok  = int(max_tokens)    if max_tokens    is not None else 900
+    temp     = float(temperature) if temperature   is not None else 0.65
+    tp       = float(top_p)       if top_p         is not None else 0.9
+    rep_pen  = float(repeat_penalty) if repeat_penalty is not None else 1.1
+
+    partial = ""
+    for chunk in llm(
+        full_prompt,
+        max_tokens=max_tok,
+        temperature=temp,
+        top_p=tp,
+        repeat_penalty=rep_pen,
+        stop=["<|im_end|>", "<|im_start|>"],
+        stream=True,
+    ):
+        partial += chunk["choices"][0]["text"]
+        yield partial
+
+# ====================== RUNTIME STATUS BADGE ======================
+def _build_status() -> str:
+    parts = []
+    if IS_HF_SPACE and not IS_LOCAL:
+        parts.append("☁️ HuggingFace Space · CPU-only")
+    else:
+        parts.append("🖥️ Local mode")
+        parts.append("⚡ GPU (CUDA)" if CUDA_AVAILABLE else "🐢 CPU-only")
+    parts.append(f"threads={N_THREADS}")
+    return "  |  ".join(parts)
+
+STATUS_LINE = _build_status()
+
+# ====================== GRADIO INTERFACE ======================
+with gr.Blocks(title="Article Q&A model") as demo:
+    gr.Markdown("# sourcing 350+ articles on change")
+    gr.Markdown(
+        "Qwen3.5-4B DoRA fine-tuned on 350+ articles on change from robertolofaro.com — "
+        "experimental on CPU-only, to test embedding methods (takes a few minutes, "
+        "no selection for the category yet) — updated as of 2026-05-05"
+    )
+    gr.Markdown(f"**Runtime:** {STATUS_LINE}")
+    gr.Markdown(
+        "**NOTAM:** by querying this model you access the articles and metadata "
+        "available on robertolofaro.com and GitHub.  "
+        "Answers reflect the article corpus only — do not treat them as advice specific to your context."
+    )
+    gr.Markdown(
+        "If, after getting an answer, you want something more contextualised, "
+        "contact a consultant (myself included)."
+    )
+
+    with gr.Row():
+        rag_mode = gr.Radio(
+            choices=list(BACKENDS.keys()),
+            value="FAISS - RAG (HNSW)",
+            label="Retrieval mode",
+        )
+        article_filter = gr.Dropdown(
+            choices=ARTICLE_LIST,
+            value="All categories",
+            label="Focus on category",
+        )
+
+    with gr.Row():
+        use_morning_news = gr.Checkbox(
+            value=False,
+            label="📰 Read MorningNews",
+            info="Supplement with the latest Morning News briefing fetched from GitHub "
+                 f"(robertolofaro/supportmaterial · {GH_NEWS_PATH}). "
+                 "Results are cached for 1 hour.",
+        )
+
+    with gr.Accordion("Advanced Generation Parameters", open=False):
+        max_tokens    = gr.Slider(256, 2048, value=900,  step=64,   label="Max Tokens")
+        temperature   = gr.Slider(0.0, 1.0,  value=0.65, step=0.05, label="Temperature")
+        top_p         = gr.Slider(0.0, 1.0,  value=0.9,  step=0.05, label="Top-p")
+        repeat_penalty = gr.Slider(1.0, 2.0, value=1.1,  step=0.05, label="Repeat Penalty")
+
+    gr.ChatInterface(
+        fn=generate_response,
+        additional_inputs=[
+            rag_mode, article_filter,
+            use_morning_news, 
+            max_tokens, temperature, top_p, repeat_penalty,
+        ],
+        cache_examples=False,   # prevents Gradio from running examples at startup
+        examples=[
+            ["What is the potential for Italy? /nothink"],
+            ["What is the potential for Turin? /nothink"],
+        ],
+    )
+
+if __name__ == "__main__":
+    # Local launch: share=False keeps it on localhost only.
+    # Set share=True if you want a temporary public Gradio tunnel.
+    demo.queue(default_concurrency_limit=1).launch(share=False)