headline generator first trial

app.py

@@ -3,7 +3,8 @@ from routes.category import router  # routes.py must be in same folder
 from routes.question import askMe
 from routes.wa_gateway import wa_router
 from dotenv import load_dotenv
-from cache_init import fetch_and_cache_articles
+import asyncio
+from nuse_modules.headlines_generator import generate_and_store_headlines
 from fastapi.middleware.cors import CORSMiddleware
 
 
@@ -23,8 +24,10 @@ app.add_middleware(
 )
 
 @app.on_event("startup")
-async def startup_event():
-    fetch_and_cache_articles()
+def fetch_and_cache_articles() -> None:
+    loop = asyncio.get_event_loop()
+    # Run in default thread-pool executor
+    loop.run_in_executor(None, generate_and_store_headlines)
 
 @app.get("/health")
 def health_check():

nuse_modules/headlines_generator.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+import datetime as _dt
+import json, os
+from typing import List, Dict
+
+from clients.redis_client import redis_client as _r              
+from nuse_modules.google_search import search_google_news       
+from models_initialization.mistral_registry import mistral_generate
+
+
+_CATEGORIES = {
+    "world":         "world news top stories",
+    "india":         "india top stories",
+    "finance":       "business finance economy today",
+    "sports":        "sports headlines today",
+    "entertainment": "entertainment celebrity movie tv",
+}
+
+_ARTICLES_PER_CAT   = 5
+_SUMMARY_TOKENS     = 120
+_REDIS_TTL_SECONDS  = 24 * 3600  
+
+def _dedupe_urls(articles: List[dict]) -> List[dict]:
+    seen = set()
+    out  = []
+    for art in articles:
+        if art["link"] not in seen:
+            seen.add(art["link"])
+            out.append(art)
+    return out
+
+
+def _summarise_article(article: dict) -> str:
+    prompt = (
+        "You are a concise news assistant. Summarise the following article "
+        "in one sentence (<=25 words). Omit source and author names.\n\n"
+        f"ARTICLE:\n{article['content']}"
+    )
+    return mistral_generate(prompt, max_new_tokens=_SUMMARY_TOKENS, temperature=0.3)
+
+
+def _redis_key(date: str, category: str) -> str:
+    return f"headlines:{date}:{category}"
+
+
+def generate_and_store_headlines(today: str | None = None) -> Dict[str, List[dict]]:
+    """
+    Fetches top articles per category, summarises them, stores in Redis,
+    and returns the full payload (useful for logging / testing).
+    """
+    date_str   = today or _dt.datetime.utcnow().strftime("%Y-%m-%d")
+    all_output = {}
+
+    for cat, query in _CATEGORIES.items():
+        print(f"[HEADLINES] {cat.title()} …")
+
+        # 1. Google -> list of {title, link, snippet, content}
+        raw_articles = search_google_news([query], num_results=_ARTICLES_PER_CAT)
+        raw_articles = _dedupe_urls(raw_articles)
+
+        # 2. Summarise each article
+        summaries = []
+        for art in raw_articles:
+            if not art["content"]:
+                continue  # skip if scraper failed
+            summary = _summarise_article(art)
+            summaries.append(
+                {
+                    "title": art["title"],
+                    "url":   art["link"],
+                    "summary": summary,
+                    "source_snippet": art["snippet"],
+                }
+            )
+
+        # 3. Store in Upstash Redis
+        redis_key = _redis_key(date_str, cat)
+        _r.set(redis_key, json.dumps(summaries), ex=_REDIS_TTL_SECONDS)
+
+        all_output[cat] = summaries
+        print(f"  ↳ stored {len(summaries)} items in Redis ({redis_key})")
+
+    return all_output