Upload folder using huggingface_hub

README.md

@@ -64,6 +64,12 @@ A multi-agent AI system that aggregates intelligence from 47+ data sources to pr
 - Domain filtering (political, economic, weather, social)
 - Floating chat UI in dashboard
 
+✅ **Trending/Velocity Detection** 🆕:
+- SQLite-based topic frequency tracking (24-hour rolling window)
+- Momentum calculation: `current_hour / avg_last_6_hours`
+- Spike alerts when topic volume > 3x baseline
+- Integrated into Combined Agent dashboard
+
 ✅ **Real-Time Dashboard** with:
 - Live Intelligence Feed
 - Floating AI Chatbox
@@ -71,6 +77,8 @@ A multi-agent AI system that aggregates intelligence from 47+ data sources to pr
 - **Live Satellite/Weather Map** (Windy.com) 🆕
 - **National Flood Threat Score** 🆕
 - **30-Year Historical Climate Analysis** 🆕
+- **Trending Topics & Spike Alerts** 🆕
+- **Enhanced Operational Indicators** 🆕 (infrastructure_health, regulatory_activity, investment_climate)
 - Operational Risk Radar
 - ML Anomaly Detection Display
 - Market Predictions with Moving Averages

frontend/app/components/intelligence/IntelligenceFeed.tsx

@@ -1,10 +1,12 @@
 import { Card } from "../ui/card";
 import { Badge } from "../ui/badge";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs";
-import { Newspaper, Cloud, TrendingUp, FileText, Radio, Globe, MapPin } from "lucide-react";
+import { Button } from "../ui/button";
+import { Newspaper, Cloud, TrendingUp, FileText, Radio, Globe, MapPin, Settings } from "lucide-react";
 import { useRogerData, RogerEvent } from "../../hooks/use-roger-data";
-import { motion } from "framer-motion";
+import { motion, AnimatePresence } from "framer-motion";
 import { useState } from "react";
+import IntelligenceSettings from "./IntelligenceSettings";
 
 const IntelligenceFeed = () => {
   const { events, isConnected } = useRogerData();
@@ -12,6 +14,9 @@ const IntelligenceFeed = () => {
   // Region toggle state (Sri Lanka / World)
   const [region, setRegion] = useState<"sri_lanka" | "world">("sri_lanka");
 
+  // Settings panel toggle
+  const [showSettings, setShowSettings] = useState(false);
+
   // ALWAYS ensure events is an array
   const safeEvents: RogerEvent[] = Array.isArray(events) ? events : [];
 
@@ -118,21 +123,46 @@ const IntelligenceFeed = () => {
           <Radio className="w-5 h-5 text-primary" />
           <h2 className="text-lg font-bold">INTELLIGENCE FEED</h2>
 
-          <span className="ml-auto text-xs font-mono text-muted-foreground">
-            {isConnected ? (
-              <span className="flex items-center gap-2">
-                <span className="w-2 h-2 rounded-full bg-success animate-pulse"></span>
-                Live
-              </span>
-            ) : (
-              <span className="flex items-center gap-2">
-                <span className="w-2 h-2 rounded-full bg-warning"></span>
-                Reconnecting...
-              </span>
-            )}
-          </span>
+          <div className="ml-auto flex items-center gap-3">
+            <Button
+              variant={showSettings ? "default" : "outline"}
+              size="sm"
+              onClick={() => setShowSettings(!showSettings)}
+              className="flex items-center gap-1"
+            >
+              <Settings className="w-4 h-4" />
+              <span className="hidden sm:inline">{showSettings ? "Hide" : "Settings"}</span>
+            </Button>
+            <span className="text-xs font-mono text-muted-foreground">
+              {isConnected ? (
+                <span className="flex items-center gap-2">
+                  <span className="w-2 h-2 rounded-full bg-success animate-pulse"></span>
+                  Live
+                </span>
+              ) : (
+                <span className="flex items-center gap-2">
+                  <span className="w-2 h-2 rounded-full bg-warning"></span>
+                  Reconnecting...
+                </span>
+              )}
+            </span>
+          </div>
         </div>
 
+        {/* SETTINGS PANEL */}
+        <AnimatePresence>
+          {showSettings && (
+            <motion.div
+              initial={{ opacity: 0, height: 0 }}
+              animate={{ opacity: 1, height: "auto" }}
+              exit={{ opacity: 0, height: 0 }}
+              className="mb-4 overflow-hidden"
+            >
+              <IntelligenceSettings />
+            </motion.div>
+          )}
+        </AnimatePresence>
+
         {/* REGION TOGGLE - Sri Lanka / World */}
         <div className="flex gap-2 mb-4 overflow-x-auto hide-scrollbar">
           <button

src/config/intel_config.json

@@ -5,5 +5,68 @@
         "linkedin": []
     },
     "user_keywords": [],
-    "user_products": []
+    "user_products": [],
+    "operational_keywords": {
+        "infrastructure": [
+            "Colombo port",
+            "Hambantota port",
+            "port strike",
+            "power outage",
+            "water shortage",
+            "fuel shortage",
+            "airport delay",
+            "customs clearance",
+            "road closure",
+            "railway disruption"
+        ],
+        "government": [
+            "cabinet decision",
+            "new policy",
+            "regulation change",
+            "tax amendment",
+            "import restriction",
+            "export ban",
+            "license requirement",
+            "central bank",
+            "budget announcement"
+        ],
+        "opportunity": [
+            "investment",
+            "expansion",
+            "new factory",
+            "job creation",
+            "export growth",
+            "tourism boost",
+            "infrastructure project",
+            "development grant",
+            "FDI",
+            "trade agreement"
+        ]
+    },
+    "alert_thresholds": {
+        "trending_momentum_min": 2.0,
+        "spike_multiplier": 3.0,
+        "high_risk_score": 0.7,
+        "high_opportunity_score": 0.6
+    },
+    "default_competitors": {
+        "telecom": {
+            "twitter": [
+                "DialogSriLanka",
+                "Mobaborang",
+                "HutchSL"
+            ],
+            "facebook": [
+                "Dialog",
+                "SLT-Mobitel",
+                "Hutch"
+            ]
+        }
+    },
+    "notes": {
+        "removed_profiles": [
+            "SLTMobitel - Twitter profile not found/restricted"
+        ],
+        "last_verified": "2025-12-08"
+    }
 }

src/nodes/combinedAgentNode.py

@@ -14,6 +14,13 @@ from typing import Dict, Any, List
 # Import storage manager for production-grade persistence
 from src.storage.storage_manager import StorageManager
 
+# Import trending detector for velocity metrics
+try:
+    from src.utils.trending_detector import get_trending_detector, record_topic_mention
+    TRENDING_ENABLED = True
+except ImportError:
+    TRENDING_ENABLED = False
+
 logger = logging.getLogger("combined_node")
 logger.setLevel(logging.INFO)
 if not logger.handlers:
@@ -473,6 +480,11 @@ JSON only:"""
             "avg_confidence": 0.0,
             "high_priority_count": 0,
             "total_events": 0,
+            "trending_topics": [],
+            "spike_alerts": [],
+            "infrastructure_health": 1.0,
+            "regulatory_activity": 0.0,
+            "investment_climate": 0.5,
             "last_updated": datetime.utcnow().isoformat()
         }
         
@@ -527,6 +539,42 @@ JSON only:"""
         snapshot["avg_confidence"] = round(avg_confidence, 3)
         snapshot["high_priority_count"] = high_priority_count
         snapshot["total_events"] = len(feed)
+        
+        # NEW: Enhanced Operational Indicators
+        # Infrastructure Health (inverted logistics friction)
+        snapshot["infrastructure_health"] = round(max(0, 1.0 - snapshot["logistics_friction"]), 3)
+        
+        # Regulatory Activity (sum of political events)
+        snapshot["regulatory_activity"] = round(len(political_scores) * 0.1, 3)
+        
+        # Investment Climate (opportunity-weighted)
+        if opportunity_scores:
+            snapshot["investment_climate"] = round(0.5 + safe_avg(opportunity_scores) * 0.5, 3)
+        
+        # NEW: Record topics for trending analysis and get current trends
+        if TRENDING_ENABLED:
+            try:
+                detector = get_trending_detector()
+                
+                # Record topics from feed
+                for item in feed:
+                    summary = item.get("summary", "")
+                    domain = item.get("target_agent", "unknown")
+                    
+                    # Extract key topic words (simplified - just use first 3 words)
+                    words = summary.split()[:5]
+                    if words:
+                        topic = " ".join(words).lower()
+                        record_topic_mention(topic, source="roger_feed", domain=domain)
+                
+                # Get trending topics and spike alerts
+                snapshot["trending_topics"] = detector.get_trending_topics(limit=5)
+                snapshot["spike_alerts"] = detector.get_spike_alerts(limit=3)
+                
+                logger.info(f"[DataRefresherAgent] Trending: {len(snapshot['trending_topics'])} topics, {len(snapshot['spike_alerts'])} spikes")
+            except Exception as e:
+                logger.warning(f"[DataRefresherAgent] Trending detection failed: {e}")
+        
         snapshot["last_updated"] = datetime.utcnow().isoformat()
         
         logger.info(f"[DataRefresherAgent] Dashboard Metrics:")

src/utils/profile_scrapers.py

@@ -45,6 +45,11 @@ def scrape_twitter_profile(username: str, max_items: int = 20):
     Fetches tweets from a specific user's profile, not search results.
     Perfect for monitoring competitor accounts, influencers, or specific business profiles.
     
+    Features:
+    - Retry logic with exponential backoff (3 attempts)
+    - Fallback to keyword search if profile fails
+    - Increased timeout (90s)
+    
     Args:
         username: Twitter username (without @)
         max_items: Maximum number of tweets to fetch
@@ -106,39 +111,79 @@ def scrape_twitter_profile(username: str, max_items: int = 20):
             
             page = context.new_page()
             
-            # Navigate to user profile
+            # Navigate to user profile with retry logic
             profile_url = f"https://x.com/{username}"
             logger.info(f"[TWITTER_PROFILE] Monitoring @{username}")
             
-            try:
-                page.goto(profile_url, timeout=60000, wait_until="domcontentloaded")
-                time.sleep(5)
-                
-                # Handle popups
-                popup_selectors = [
-                    "[data-testid='app-bar-close']",
-                    "[aria-label='Close']",
-                    "button:has-text('Not now')",
-                ]
-                for selector in popup_selectors:
+            max_retries = 3
+            navigation_success = False
+            last_error = None
+            
+            for attempt in range(max_retries):
+                try:
+                    # Exponential backoff: 0, 2, 4 seconds
+                    if attempt > 0:
+                        wait_time = 2 ** attempt
+                        logger.info(f"[TWITTER_PROFILE] Retry {attempt + 1}/{max_retries} after {wait_time}s...")
+                        time.sleep(wait_time)
+                    
+                    # Increased timeout from 60s to 90s, changed to networkidle
+                    page.goto(profile_url, timeout=90000, wait_until="networkidle")
+                    time.sleep(5)
+                    
+                    # Handle popups
+                    popup_selectors = [
+                        "[data-testid='app-bar-close']",
+                        "[aria-label='Close']",
+                        "button:has-text('Not now')",
+                    ]
+                    for selector in popup_selectors:
+                        try:
+                            if page.locator(selector).count() > 0 and page.locator(selector).first.is_visible():
+                                page.locator(selector).first.click()
+                                time.sleep(1)
+                        except:
+                            pass
+                    
+                    # Wait for tweets to load
                     try:
-                        if page.locator(selector).count() > 0 and page.locator(selector).first.is_visible():
-                            page.locator(selector).first.click()
-                            time.sleep(1)
+                        page.wait_for_selector("article[data-testid='tweet']", timeout=20000)
+                        logger.info(f"[TWITTER_PROFILE] Loaded {username}'s profile")
+                        navigation_success = True
+                        break
                     except:
-                        pass
+                        last_error = f"Could not load tweets for @{username}"
+                        logger.warning(f"[TWITTER_PROFILE] {last_error}, attempt {attempt + 1}/{max_retries}")
+                        continue
+                        
+                except Exception as e:
+                    last_error = str(e)
+                    logger.warning(f"[TWITTER_PROFILE] Navigation failed on attempt {attempt + 1}: {e}")
+                    continue
+            
+            # If profile scraping failed after all retries, try fallback to keyword search
+            if not navigation_success:
+                logger.warning(f"[TWITTER_PROFILE] Profile scraping failed, falling back to keyword search for '{username}'")
+                browser.close()
                 
-                # Wait for tweets to load
+                # Fallback: use keyword search instead
                 try:
-                    page.wait_for_selector("article[data-testid='tweet']", timeout=15000)
-                    logger.info(f"[TWITTER_PROFILE] Loaded {username}'s profile")
-                except:
-                    logger.error(f"[TWITTER_PROFILE] Could not load tweets for @{username}")
-                    return json.dumps({"error": f"Profile not found or private: @{username}"}, default=str)
+                    from src.utils.utils import scrape_twitter
+                    fallback_result = scrape_twitter.invoke({"query": username, "max_items": max_items})
+                    fallback_data = json.loads(fallback_result) if isinstance(fallback_result, str) else fallback_result
+                    
+                    if "error" not in fallback_data:
+                        fallback_data["fallback_used"] = True
+                        fallback_data["original_error"] = last_error
+                        fallback_data["note"] = f"Used keyword search as fallback for @{username}"
+                        return json.dumps(fallback_data, default=str)
+                except Exception as fallback_error:
+                    logger.error(f"[TWITTER_PROFILE] Fallback also failed: {fallback_error}")
                 
-            except Exception as e:
-                logger.error(f"[TWITTER_PROFILE] Navigation failed: {e}")
-                return json.dumps({"error": str(e)}, default=str)
+                return json.dumps({
+                    "error": last_error or f"Profile not found or private: @{username}",
+                    "fallback_attempted": True
+                }, default=str)
             
             # Check if logged in
             if "login" in page.url:

src/utils/trending_detector.py

@@ -0,0 +1,325 @@
+"""
+src/utils/trending_detector.py
+Trending/Velocity Detection Module for Roger
+
+Tracks topic mention frequency over time to detect:
+- Topics gaining traction (momentum)
+- Sudden volume spikes (alerts)
+- Trending topics across the system
+
+Uses SQLite for persistence.
+"""
+import os
+import json
+import sqlite3
+import hashlib
+import logging
+from datetime import datetime, timedelta
+from typing import List, Dict, Any, Optional, Tuple
+from pathlib import Path
+
+logger = logging.getLogger("Roger.trending")
+
+# Default database path
+DEFAULT_DB_PATH = os.path.join(
+    os.path.dirname(__file__), "..", "..", "data", "trending.db"
+)
+
+
+class TrendingDetector:
+    """
+    Detects trending topics and velocity spikes.
+    
+    Features:
+    - Records topic mentions with timestamps
+    - Calculates momentum (current_hour / avg_last_6_hours)
+    - Detects spikes (>3x normal volume in 1 hour)
+    - Returns trending topics for dashboard display
+    """
+    
+    def __init__(self, db_path: str = None, spike_threshold: float = 3.0, momentum_threshold: float = 2.0):
+        """
+        Initialize the TrendingDetector.
+        
+        Args:
+            db_path: Path to SQLite database (default: data/trending.db)
+            spike_threshold: Multiplier for spike detection (default: 3x)
+            momentum_threshold: Minimum momentum to be considered trending (default: 2.0)
+        """
+        self.db_path = db_path or DEFAULT_DB_PATH
+        self.spike_threshold = spike_threshold
+        self.momentum_threshold = momentum_threshold
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
+        
+        # Initialize database
+        self._init_db()
+        logger.info(f"[TrendingDetector] Initialized with db: {self.db_path}")
+    
+    def _init_db(self):
+        """Create tables if they don't exist"""
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS topic_mentions (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    topic TEXT NOT NULL,
+                    topic_hash TEXT NOT NULL,
+                    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+                    source TEXT,
+                    domain TEXT
+                )
+            """)
+            conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_topic_hash ON topic_mentions(topic_hash)
+            """)
+            conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_timestamp ON topic_mentions(timestamp)
+            """)
+            
+            # Hourly aggregates for faster queries
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS hourly_counts (
+                    topic_hash TEXT NOT NULL,
+                    hour_bucket TEXT NOT NULL,
+                    count INTEGER DEFAULT 1,
+                    topic TEXT,
+                    PRIMARY KEY (topic_hash, hour_bucket)
+                )
+            """)
+            conn.commit()
+    
+    def _topic_hash(self, topic: str) -> str:
+        """Generate a hash for a topic (normalized lowercase)"""
+        normalized = topic.lower().strip()
+        return hashlib.md5(normalized.encode()).hexdigest()[:12]
+    
+    def _get_hour_bucket(self, dt: datetime = None) -> str:
+        """Get the hour bucket string (YYYY-MM-DD-HH)"""
+        dt = dt or datetime.utcnow()
+        return dt.strftime("%Y-%m-%d-%H")
+    
+    def record_mention(
+        self, 
+        topic: str, 
+        source: str = None, 
+        domain: str = None,
+        timestamp: datetime = None
+    ):
+        """
+        Record a topic mention.
+        
+        Args:
+            topic: The topic/keyword mentioned
+            source: Source of the mention (e.g., 'twitter', 'news')
+            domain: Domain (e.g., 'political', 'economical')
+            timestamp: When the mention occurred (default: now)
+        """
+        topic_hash = self._topic_hash(topic)
+        ts = timestamp or datetime.utcnow()
+        hour_bucket = self._get_hour_bucket(ts)
+        
+        with sqlite3.connect(self.db_path) as conn:
+            # Insert mention
+            conn.execute("""
+                INSERT INTO topic_mentions (topic, topic_hash, timestamp, source, domain)
+                VALUES (?, ?, ?, ?, ?)
+            """, (topic.lower().strip(), topic_hash, ts.isoformat(), source, domain))
+            
+            # Update hourly aggregate
+            conn.execute("""
+                INSERT INTO hourly_counts (topic_hash, hour_bucket, count, topic)
+                VALUES (?, ?, 1, ?)
+                ON CONFLICT(topic_hash, hour_bucket) DO UPDATE SET count = count + 1
+            """, (topic_hash, hour_bucket, topic.lower().strip()))
+            
+            conn.commit()
+    
+    def record_mentions_batch(self, mentions: List[Dict[str, Any]]):
+        """
+        Record multiple mentions at once.
+        
+        Args:
+            mentions: List of dicts with keys: topic, source, domain, timestamp
+        """
+        for mention in mentions:
+            self.record_mention(
+                topic=mention.get("topic", ""),
+                source=mention.get("source"),
+                domain=mention.get("domain"),
+                timestamp=mention.get("timestamp")
+            )
+    
+    def get_momentum(self, topic: str) -> float:
+        """
+        Calculate momentum for a topic.
+        
+        Momentum = mentions_in_current_hour / avg_mentions_in_last_6_hours
+        
+        Returns:
+            Momentum value (1.0 = normal, >2.0 = trending, >3.0 = spike)
+        """
+        topic_hash = self._topic_hash(topic)
+        now = datetime.utcnow()
+        current_hour = self._get_hour_bucket(now)
+        
+        with sqlite3.connect(self.db_path) as conn:
+            # Get current hour count
+            result = conn.execute("""
+                SELECT count FROM hourly_counts 
+                WHERE topic_hash = ? AND hour_bucket = ?
+            """, (topic_hash, current_hour)).fetchone()
+            current_count = result[0] if result else 0
+            
+            # Get average of last 6 hours
+            past_hours = []
+            for i in range(1, 7):
+                past_dt = now - timedelta(hours=i)
+                past_hours.append(self._get_hour_bucket(past_dt))
+            
+            placeholders = ",".join(["?" for _ in past_hours])
+            result = conn.execute(f"""
+                SELECT AVG(count) FROM hourly_counts 
+                WHERE topic_hash = ? AND hour_bucket IN ({placeholders})
+            """, [topic_hash] + past_hours).fetchone()
+            avg_count = result[0] if result and result[0] else 0.1  # Avoid division by zero
+            
+            return current_count / avg_count if avg_count > 0 else current_count
+    
+    def is_spike(self, topic: str, window_hours: int = 1) -> bool:
+        """
+        Check if a topic is experiencing a spike.
+        
+        A spike is when current volume > spike_threshold * normal volume.
+        """
+        momentum = self.get_momentum(topic)
+        return momentum >= self.spike_threshold
+    
+    def get_trending_topics(self, limit: int = 10) -> List[Dict[str, Any]]:
+        """
+        Get topics with momentum above threshold.
+        
+        Returns:
+            List of trending topics with their momentum values
+        """
+        now = datetime.utcnow()
+        current_hour = self._get_hour_bucket(now)
+        
+        trending = []
+        
+        with sqlite3.connect(self.db_path) as conn:
+            # Get all topics mentioned in current hour
+            results = conn.execute("""
+                SELECT DISTINCT topic, topic_hash, count 
+                FROM hourly_counts 
+                WHERE hour_bucket = ?
+                ORDER BY count DESC
+                LIMIT 50
+            """, (current_hour,)).fetchall()
+            
+            for topic, topic_hash, count in results:
+                momentum = self.get_momentum(topic)
+                
+                if momentum >= self.momentum_threshold:
+                    trending.append({
+                        "topic": topic,
+                        "momentum": round(momentum, 2),
+                        "mentions_this_hour": count,
+                        "is_spike": momentum >= self.spike_threshold,
+                        "severity": "high" if momentum >= 5 else "medium" if momentum >= 3 else "low"
+                    })
+        
+        # Sort by momentum descending
+        trending.sort(key=lambda x: x["momentum"], reverse=True)
+        return trending[:limit]
+    
+    def get_spike_alerts(self, limit: int = 5) -> List[Dict[str, Any]]:
+        """
+        Get topics with spike alerts (>3x normal volume).
+        
+        Returns:
+            List of spike alerts
+        """
+        return [t for t in self.get_trending_topics(limit=50) if t["is_spike"]][:limit]
+    
+    def get_topic_history(self, topic: str, hours: int = 24) -> List[Dict[str, Any]]:
+        """
+        Get hourly mention counts for a topic.
+        
+        Args:
+            topic: Topic to get history for
+            hours: Number of hours to look back
+        
+        Returns:
+            List of hourly counts
+        """
+        topic_hash = self._topic_hash(topic)
+        now = datetime.utcnow()
+        
+        history = []
+        with sqlite3.connect(self.db_path) as conn:
+            for i in range(hours):
+                hour_dt = now - timedelta(hours=i)
+                hour_bucket = self._get_hour_bucket(hour_dt)
+                
+                result = conn.execute("""
+                    SELECT count FROM hourly_counts 
+                    WHERE topic_hash = ? AND hour_bucket = ?
+                """, (topic_hash, hour_bucket)).fetchone()
+                
+                history.append({
+                    "hour": hour_bucket,
+                    "count": result[0] if result else 0
+                })
+        
+        return list(reversed(history))  # Oldest first
+    
+    def cleanup_old_data(self, days: int = 7):
+        """
+        Remove data older than specified days.
+        
+        Args:
+            days: Number of days to keep
+        """
+        cutoff = datetime.utcnow() - timedelta(days=days)
+        cutoff_str = cutoff.isoformat()
+        cutoff_bucket = self._get_hour_bucket(cutoff)
+        
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute("""
+                DELETE FROM topic_mentions WHERE timestamp < ?
+            """, (cutoff_str,))
+            conn.execute("""
+                DELETE FROM hourly_counts WHERE hour_bucket < ?
+            """, (cutoff_bucket,))
+            conn.commit()
+        
+        logger.info(f"[TrendingDetector] Cleaned up data older than {days} days")
+
+
+# Singleton instance for easy access
+_trending_detector = None
+
+
+def get_trending_detector() -> TrendingDetector:
+    """Get the global TrendingDetector instance"""
+    global _trending_detector
+    if _trending_detector is None:
+        _trending_detector = TrendingDetector()
+    return _trending_detector
+
+
+# Convenience functions
+def record_topic_mention(topic: str, source: str = None, domain: str = None):
+    """Record a single topic mention"""
+    get_trending_detector().record_mention(topic, source, domain)
+
+
+def get_trending_now(limit: int = 10) -> List[Dict[str, Any]]:
+    """Get current trending topics"""
+    return get_trending_detector().get_trending_topics(limit)
+
+
+def get_spikes() -> List[Dict[str, Any]]:
+    """Get current spike alerts"""
+    return get_trending_detector().get_spike_alerts()

test_all_tools.py

@@ -0,0 +1,276 @@
+"""
+test_all_tools.py
+Comprehensive test script for all Roger agentic AI tools
+Runs each tool and validates output format
+"""
+import json
+import sys
+import os
+
+# Force UTF-8 output on Windows
+if sys.platform == 'win32':
+    import io
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+# Add project root to path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from datetime import datetime
+
+def test_tool(name, func, *args, **kwargs):
+    """Test a tool and report results"""
+    print(f"\n{'='*60}")
+    print(f"[TEST] {name}")
+    print(f"{'='*60}")
+    try:
+        result = func(*args, **kwargs)
+        
+        # Parse result if it's JSON string
+        if isinstance(result, str):
+            try:
+                parsed = json.loads(result)
+                if "error" in parsed:
+                    print(f"[WARN] TOOL RETURNED ERROR: {parsed['error']}")
+                    if "solution" in parsed:
+                        print(f"   [TIP] Solution: {parsed['solution']}")
+                    return {"status": "error", "error": parsed["error"]}
+                else:
+                    print(f"[OK] SUCCESS")
+                    # Print sample of results
+                    if "results" in parsed:
+                        print(f"   [DATA] Results count: {len(parsed['results'])}")
+                        if parsed['results'] and len(parsed['results']) > 0:
+                            print(f"   [SAMPLE] {str(parsed['results'][0])[:200]}...")
+                    elif isinstance(parsed, dict):
+                        for key in list(parsed.keys())[:3]:
+                            val = str(parsed[key])[:100]
+                            print(f"   - {key}: {val}...")
+                    return {"status": "success", "data": parsed}
+            except json.JSONDecodeError:
+                print(f"[OK] SUCCESS (non-JSON response)")
+                print(f"   [SAMPLE] {result[:200]}...")
+                return {"status": "success", "data": result}
+        else:
+            print(f"[OK] SUCCESS")
+            print(f"   [TYPE] Response type: {type(result)}")
+            return {"status": "success", "data": result}
+            
+    except Exception as e:
+        print(f"[FAIL] FAILED: {e}")
+        return {"status": "failed", "error": str(e)}
+
+def main():
+    results = {}
+    
+    print("\n" + "="*70)
+    print("[START] ROGER AGENTIC AI - COMPREHENSIVE TOOL TESTING")
+    print(f"   Started: {datetime.now().isoformat()}")
+    print("="*70)
+    
+    # =====================================================
+    # 1. WEATHER & FLOOD TOOLS (No session required)
+    # =====================================================
+    print("\n\n[CATEGORY] WEATHER & FLOOD TOOLS")
+    print("-"*50)
+    
+    try:
+        from src.utils.utils import tool_dmc_alerts
+        results["tool_dmc_alerts"] = test_tool("tool_dmc_alerts", tool_dmc_alerts)
+    except ImportError as e:
+        print(f"[FAIL] Import error: {e}")
+        results["tool_dmc_alerts"] = {"status": "import_error", "error": str(e)}
+    
+    try:
+        from src.utils.utils import tool_weather_nowcast
+        results["tool_weather_nowcast"] = test_tool("tool_weather_nowcast", tool_weather_nowcast)
+    except ImportError as e:
+        print(f"[FAIL] Import error: {e}")
+        results["tool_weather_nowcast"] = {"status": "import_error", "error": str(e)}
+    
+    try:
+        from src.utils.utils import tool_rivernet_status
+        results["tool_rivernet_status"] = test_tool("tool_rivernet_status", tool_rivernet_status)
+    except ImportError as e:
+        print(f"[FAIL] Import error: {e}")
+        results["tool_rivernet_status"] = {"status": "import_error", "error": str(e)}
+    
+    try:
+        from src.utils.utils import tool_district_weather
+        results["tool_district_weather"] = test_tool("tool_district_weather", tool_district_weather, "colombo")
+    except ImportError as e:
+        print(f"[FAIL] Import error: {e}")
+        results["tool_district_weather"] = {"status": "import_error", "error": str(e)}
+    
+    # =====================================================
+    # 2. NEWS & OFFICIAL SOURCES (No session required)
+    # =====================================================
+    print("\n\n[CATEGORY] NEWS & OFFICIAL SOURCES")
+    print("-"*50)
+    
+    try:
+        from src.utils.tool_factory import create_tool_set
+        tools = create_tool_set(include_profile_scrapers=False)
+        
+        # Local News
+        local_news = tools.get("scrape_local_news")
+        if local_news:
+            results["scrape_local_news"] = test_tool("scrape_local_news", local_news.invoke, {"keywords": ["sri lanka"], "max_articles": 5})
+        else:
+            results["scrape_local_news"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+        
+        # CSE Stock Data
+        cse_tool = tools.get("scrape_cse_stock_data")
+        if cse_tool:
+            results["scrape_cse_stock_data"] = test_tool("scrape_cse_stock_data", cse_tool.invoke, {"symbol": "ASPI", "period": "1d"})
+        else:
+            results["scrape_cse_stock_data"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+        
+        # Government Gazette
+        gazette_tool = tools.get("scrape_government_gazette")
+        if gazette_tool:
+            results["scrape_government_gazette"] = test_tool("scrape_government_gazette", gazette_tool.invoke, {"keywords": None, "max_items": 5})
+        else:
+            results["scrape_government_gazette"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+        
+        # Parliament Minutes
+        parliament_tool = tools.get("scrape_parliament_minutes")
+        if parliament_tool:
+            results["scrape_parliament_minutes"] = test_tool("scrape_parliament_minutes", parliament_tool.invoke, {"keywords": None, "max_items": 5})
+        else:
+            results["scrape_parliament_minutes"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+        
+        # Reddit (no session needed)
+        reddit_tool = tools.get("scrape_reddit")
+        if reddit_tool:
+            results["scrape_reddit"] = test_tool("scrape_reddit", reddit_tool.invoke, {"keywords": ["sri lanka"], "limit": 5})
+        else:
+            results["scrape_reddit"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+            
+    except Exception as e:
+        print(f"[FAIL] ToolSet creation error: {e}")
+        results["tool_factory"] = {"status": "failed", "error": str(e)}
+    
+    # =====================================================
+    # 3. SOCIAL MEDIA TOOLS (Session required)
+    # =====================================================
+    print("\n\n[CATEGORY] SOCIAL MEDIA TOOLS (Session Required)")
+    print("-"*50)
+    
+    try:
+        from src.utils.tool_factory import create_tool_set
+        tools = create_tool_set(include_profile_scrapers=True)
+        
+        # Twitter Search
+        twitter_tool = tools.get("scrape_twitter")
+        if twitter_tool:
+            results["scrape_twitter"] = test_tool("scrape_twitter", twitter_tool.invoke, {"query": "sri lanka", "max_items": 3})
+        else:
+            results["scrape_twitter"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+        
+        # Facebook Search
+        fb_tool = tools.get("scrape_facebook")
+        if fb_tool:
+            results["scrape_facebook"] = test_tool("scrape_facebook", fb_tool.invoke, {"keywords": ["sri lanka"], "max_items": 3})
+        else:
+            results["scrape_facebook"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+        
+        # LinkedIn Search
+        linkedin_tool = tools.get("scrape_linkedin")
+        if linkedin_tool:
+            results["scrape_linkedin"] = test_tool("scrape_linkedin", linkedin_tool.invoke, {"keywords": ["sri lanka"], "max_items": 3})
+        else:
+            results["scrape_linkedin"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+        
+        # Instagram Search
+        instagram_tool = tools.get("scrape_instagram")
+        if instagram_tool:
+            results["scrape_instagram"] = test_tool("scrape_instagram", instagram_tool.invoke, {"keywords": ["srilanka"], "max_items": 3})
+        else:
+            results["scrape_instagram"] = {"status": "not_found", "error": "Tool not found in ToolSet"}
+            
+    except Exception as e:
+        print(f"[FAIL] Social media tools error: {e}")
+    
+    # =====================================================
+    # 4. PROFILE SCRAPERS (Session required)
+    # =====================================================
+    print("\n\n[CATEGORY] PROFILE SCRAPERS (Session Required)")
+    print("-"*50)
+    
+    try:
+        from src.utils.profile_scrapers import scrape_twitter_profile, scrape_facebook_profile
+        
+        # Twitter Profile
+        results["scrape_twitter_profile"] = test_tool("scrape_twitter_profile", scrape_twitter_profile.invoke, {"username": "SLTMobitel", "max_items": 3})
+        
+        # Facebook Profile  
+        results["scrape_facebook_profile"] = test_tool("scrape_facebook_profile", scrape_facebook_profile.invoke, {"profile_url": "https://www.facebook.com/DialogAxiata", "max_items": 3})
+        
+    except Exception as e:
+        print(f"[FAIL] Profile scrapers error: {e}")
+    
+    # =====================================================
+    # SUMMARY REPORT
+    # =====================================================
+    print("\n\n" + "="*70)
+    print("[SUMMARY] TEST RESULTS")
+    print("="*70)
+    
+    success_count = 0
+    error_count = 0
+    session_issues = []
+    other_errors = []
+    
+    for tool_name, result in results.items():
+        status = result.get("status", "unknown")
+        if status == "success":
+            success_count += 1
+            print(f"[OK] {tool_name}: SUCCESS")
+        elif status == "error":
+            error = result.get("error", "Unknown error")
+            if "session" in error.lower() or "Session" in error:
+                session_issues.append(tool_name)
+                print(f"[SESSION] {tool_name}: SESSION ISSUE - {error[:50]}")
+            else:
+                other_errors.append((tool_name, error))
+                print(f"[WARN] {tool_name}: ERROR - {error[:50]}")
+            error_count += 1
+        else:
+            error = result.get("error", "Unknown")
+            other_errors.append((tool_name, error))
+            print(f"[FAIL] {tool_name}: {status.upper()} - {error[:50]}")
+            error_count += 1
+    
+    print(f"\n[TOTALS]:")
+    print(f"   [OK] Successful: {success_count}")
+    print(f"   [FAIL] Errors: {error_count}")
+    
+    if session_issues:
+        print(f"\n[SESSION] TOOLS NEEDING SESSION REFRESH:")
+        for tool in session_issues:
+            print(f"   - {tool}")
+    
+    if other_errors:
+        print(f"\n[WARN] TOOLS WITH OTHER ERRORS:")
+        for tool, error in other_errors:
+            print(f"   - {tool}: {error[:80]}")
+    
+    # Save results to file
+    with open("tool_test_results.json", "w", encoding="utf-8") as f:
+        json.dump({
+            "timestamp": datetime.now().isoformat(),
+            "summary": {
+                "success": success_count,
+                "errors": error_count,
+                "session_issues": session_issues,
+                "other_errors": [t[0] for t in other_errors]
+            },
+            "details": {k: {"status": v.get("status"), "error": v.get("error")} for k, v in results.items()}
+        }, f, indent=2)
+    print(f"\n[SAVED] Results saved to: tool_test_results.json")
+    
+    return results
+
+if __name__ == "__main__":
+    main()
+

test_ml_pipelines.py

@@ -0,0 +1,231 @@
+"""
+test_ml_pipelines.py
+Test script to verify all 4 ML pipelines are working correctly
+"""
+import sys
+import os
+import io
+
+# Force UTF-8 output on Windows
+if sys.platform == 'win32':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+# Add project root to path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+print("="*70)
+print("[ML PIPELINE TESTING]")
+print("="*70)
+
+results = {}
+
+# =============================================================================
+# 1. ANOMALY DETECTION PIPELINE
+# =============================================================================
+print("\n" + "="*60)
+print("[1] ANOMALY DETECTION PIPELINE")
+print("="*60)
+
+try:
+    # Check if model exists
+    from pathlib import Path
+    model_dir = Path(__file__).parent / "models" / "anomaly-detection" / "output"
+    models_found = list(model_dir.glob("*.joblib")) if model_dir.exists() else []
+    
+    if models_found:
+        print(f"[OK] Found {len(models_found)} trained models:")
+        for m in models_found[:3]:
+            print(f"   - {m.name}")
+        
+        # Try to load and run prediction
+        from models.anomaly_detection.src.utils.vectorizer import get_vectorizer
+        vectorizer = get_vectorizer()
+        print(f"[OK] Vectorizer loaded")
+        
+        import joblib
+        model = joblib.load(models_found[0])
+        print(f"[OK] Model loaded: {models_found[0].name}")
+        
+        # Test prediction
+        test_text = "Breaking news: Major political announcement in Colombo"
+        vector = vectorizer.vectorize(test_text, "en")
+        prediction = model.predict([vector])[0]
+        score = -model.decision_function([vector])[0] if hasattr(model, 'decision_function') else 0
+        
+        print(f"[OK] Test prediction: is_anomaly={prediction==-1}, score={score:.3f}")
+        results["anomaly_detection"] = {"status": "success", "models": len(models_found)}
+    else:
+        print("[WARN] No trained models found. Run training first.")
+        print("   Command: python models/anomaly-detection/main.py --mode train")
+        results["anomaly_detection"] = {"status": "not_trained"}
+        
+except Exception as e:
+    print(f"[FAIL] Anomaly Detection error: {e}")
+    results["anomaly_detection"] = {"status": "error", "error": str(e)}
+
+# =============================================================================
+# 2. WEATHER PREDICTION PIPELINE
+# =============================================================================
+print("\n" + "="*60)
+print("[2] WEATHER PREDICTION PIPELINE")
+print("="*60)
+
+try:
+    from pathlib import Path
+    weather_model_dir = Path(__file__).parent / "models" / "weather-prediction" / "artifacts" / "models"
+    weather_models = list(weather_model_dir.glob("*.h5")) if weather_model_dir.exists() else []
+    
+    predictions_dir = Path(__file__).parent / "models" / "weather-prediction" / "output" / "predictions"
+    prediction_files = list(predictions_dir.glob("*.json")) if predictions_dir.exists() else []
+    
+    if weather_models:
+        print(f"[OK] Found {len(weather_models)} trained LSTM models:")
+        for m in weather_models[:5]:
+            print(f"   - {m.name}")
+        
+        # Check for predictions
+        if prediction_files:
+            import json
+            latest = max(prediction_files, key=lambda p: p.stat().st_mtime)
+            with open(latest) as f:
+                preds = json.load(f)
+            districts = preds.get("districts", {})
+            print(f"[OK] Found predictions for {len(districts)} districts")
+            print(f"   Latest prediction date: {preds.get('prediction_date', 'N/A')}")
+            
+            # Show sample
+            if districts:
+                sample_district = list(districts.keys())[0]
+                sample = districts[sample_district]
+                print(f"   Sample ({sample_district}):")
+                print(f"     - Temp: {sample.get('temp_max', 'N/A')}C - {sample.get('temp_min', 'N/A')}C")
+                print(f"     - Rain: {sample.get('rainfall_mm', 'N/A')}mm")
+            
+            results["weather_prediction"] = {"status": "success", "models": len(weather_models), "districts": len(districts)}
+        else:
+            print("[WARN] No prediction files found. Run predictor.")
+            results["weather_prediction"] = {"status": "models_only", "models": len(weather_models)}
+    else:
+        print("[WARN] No trained models found")
+        print("   Command: python models/weather-prediction/main.py --mode train")
+        results["weather_prediction"] = {"status": "not_trained"}
+        
+except Exception as e:
+    print(f"[FAIL] Weather Prediction error: {e}")
+    results["weather_prediction"] = {"status": "error", "error": str(e)}
+
+# =============================================================================
+# 3. CURRENCY PREDICTION PIPELINE
+# =============================================================================
+print("\n" + "="*60)
+print("[3] CURRENCY PREDICTION PIPELINE (USD/LKR)")
+print("="*60)
+
+try:
+    from pathlib import Path
+    currency_model_dir = Path(__file__).parent / "models" / "currency-volatility-prediction" / "artifacts" / "models"
+    currency_model = currency_model_dir / "gru_usd_lkr.h5" if currency_model_dir.exists() else None
+    
+    predictions_dir = Path(__file__).parent / "models" / "currency-volatility-prediction" / "output" / "predictions"
+    prediction_files = list(predictions_dir.glob("*.json")) if predictions_dir.exists() else []
+    
+    if currency_model and currency_model.exists():
+        print(f"[OK] Found GRU model: {currency_model.name}")
+        
+        # Check for predictions
+        if prediction_files:
+            import json
+            latest = max(prediction_files, key=lambda p: p.stat().st_mtime)
+            with open(latest) as f:
+                pred = json.load(f)
+            
+            print(f"[OK] Latest prediction found:")
+            print(f"   - Current Rate: {pred.get('current_rate', 'N/A')} LKR")
+            print(f"   - Predicted: {pred.get('predicted_rate', 'N/A')} LKR")
+            print(f"   - Change: {pred.get('change_percent', 'N/A')}%")
+            print(f"   - Direction: {pred.get('direction', 'N/A')}")
+            
+            results["currency_prediction"] = {"status": "success", "rate": pred.get("predicted_rate")}
+        else:
+            print("[WARN] No prediction files found")
+            results["currency_prediction"] = {"status": "model_only"}
+    else:
+        print("[WARN] No trained model found")
+        print("   Command: python models/currency-volatility-prediction/main.py --mode train")
+        results["currency_prediction"] = {"status": "not_trained"}
+        
+except Exception as e:
+    print(f"[FAIL] Currency Prediction error: {e}")
+    results["currency_prediction"] = {"status": "error", "error": str(e)}
+
+# =============================================================================
+# 4. STOCK PRICE PREDICTION PIPELINE
+# =============================================================================
+print("\n" + "="*60)
+print("[4] STOCK PRICE PREDICTION PIPELINE")
+print("="*60)
+
+try:
+    from pathlib import Path
+    stock_model_dir = Path(__file__).parent / "models" / "stock-price-prediction" / "artifacts" / "models"
+    stock_models = list(stock_model_dir.glob("*.h5")) if stock_model_dir.exists() else []
+    
+    predictions_dir = Path(__file__).parent / "models" / "stock-price-prediction" / "output" / "predictions"
+    prediction_files = list(predictions_dir.glob("*.json")) if predictions_dir.exists() else []
+    
+    if stock_models:
+        print(f"[OK] Found {len(stock_models)} stock models:")
+        for m in stock_models[:5]:
+            print(f"   - {m.name}")
+        
+        # Check for predictions
+        if prediction_files:
+            import json
+            latest = max(prediction_files, key=lambda p: p.stat().st_mtime)
+            with open(latest) as f:
+                preds = json.load(f)
+            
+            stocks = preds.get("stocks", preds.get("predictions", {}))
+            print(f"[OK] Found predictions for {len(stocks)} stocks")
+            
+            # Show sample
+            if stocks:
+                sample_stock = list(stocks.keys())[0] if isinstance(stocks, dict) else stocks[0]
+                if isinstance(stocks, dict):
+                    sample = stocks[sample_stock]
+                    print(f"   Sample ({sample_stock}):")
+                    print(f"     - Current: {sample.get('current_price', 'N/A')}")
+                    print(f"     - Predicted: {sample.get('predicted_price', 'N/A')}")
+            
+            results["stock_prediction"] = {"status": "success", "models": len(stock_models), "stocks": len(stocks)}
+        else:
+            print("[WARN] No prediction files found")
+            results["stock_prediction"] = {"status": "models_only", "models": len(stock_models)}
+    else:
+        print("[WARN] No trained models found")
+        print("   Command: python models/stock-price-prediction/main.py --mode train")
+        results["stock_prediction"] = {"status": "not_trained"}
+        
+except Exception as e:
+    print(f"[FAIL] Stock Prediction error: {e}")
+    results["stock_prediction"] = {"status": "error", "error": str(e)}
+
+# =============================================================================
+# SUMMARY
+# =============================================================================
+print("\n" + "="*70)
+print("[SUMMARY] ML PIPELINE STATUS")
+print("="*70)
+
+for pipeline, result in results.items():
+    status = result.get("status", "unknown")
+    if status == "success":
+        print(f"[OK] {pipeline}: Working")
+    elif status == "not_trained":
+        print(f"[WARN] {pipeline}: Not trained yet")
+    elif status in ["model_only", "models_only"]:
+        print(f"[WARN] {pipeline}: Model exists, no recent predictions")
+    else:
+        print(f"[FAIL] {pipeline}: {result.get('error', status)}")
+
+print("="*70)

test_news_tools.py

@@ -0,0 +1,117 @@
+"""
+test_news_tools.py
+Test for news and official source tools (no social media sessions required)
+"""
+import sys
+import os
+import io
+import json
+
+# Force UTF-8 output
+if sys.platform == 'win32':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+print("="*60)
+print("[TEST] NEWS & OFFICIAL SOURCE TOOLS")
+print("="*60)
+
+try:
+    from src.utils.tool_factory import create_tool_set
+    print("[OK] Tool factory imported")
+    tools = create_tool_set(include_profile_scrapers=False)
+    print("[OK] ToolSet created")
+except Exception as e:
+    print(f"[FAIL] Could not create ToolSet: {e}")
+    sys.exit(1)
+
+# Test 1: Local News
+print("\n[1] Testing scrape_local_news...")
+try:
+    local_news = tools.get("scrape_local_news")
+    if local_news:
+        result = local_news.invoke({"keywords": ["sri lanka"], "max_articles": 3})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if "error" in parsed:
+            print(f"[WARN] Local News returned error: {parsed['error']}")
+        else:
+            count = len(parsed.get('results', []))
+            print(f"[OK] Local News: {count} articles fetched")
+            if count > 0:
+                print(f"   Sample: {str(parsed['results'][0])[:150]}...")
+    else:
+        print("[WARN] scrape_local_news not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] Local News: {e}")
+
+# Test 2: CSE Stock Data
+print("\n[2] Testing scrape_cse_stock_data...")
+try:
+    cse_tool = tools.get("scrape_cse_stock_data")
+    if cse_tool:
+        result = cse_tool.invoke({"symbol": "ASPI", "period": "1d"})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if "error" in parsed:
+            print(f"[WARN] CSE Stock returned error: {parsed['error']}")
+        else:
+            print(f"[OK] CSE Stock: {str(parsed)[:200]}...")
+    else:
+        print("[WARN] scrape_cse_stock_data not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] CSE Stock: {e}")
+
+# Test 3: Government Gazette
+print("\n[3] Testing scrape_government_gazette...")
+try:
+    gazette_tool = tools.get("scrape_government_gazette")
+    if gazette_tool:
+        result = gazette_tool.invoke({"keywords": None, "max_items": 3})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if "error" in parsed:
+            print(f"[WARN] Gazette returned error: {parsed['error']}")
+        else:
+            count = len(parsed.get('results', []))
+            print(f"[OK] Gazette: {count} items fetched")
+    else:
+        print("[WARN] scrape_government_gazette not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] Gazette: {e}")
+
+# Test 4: Parliament Minutes
+print("\n[4] Testing scrape_parliament_minutes...")
+try:
+    parliament_tool = tools.get("scrape_parliament_minutes")
+    if parliament_tool:
+        result = parliament_tool.invoke({"keywords": None, "max_items": 3})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if "error" in parsed:
+            print(f"[WARN] Parliament returned error: {parsed['error']}")
+        else:
+            count = len(parsed.get('results', []))
+            print(f"[OK] Parliament: {count} items fetched")
+    else:
+        print("[WARN] scrape_parliament_minutes not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] Parliament: {e}")
+
+# Test 5: Reddit (no auth needed)
+print("\n[5] Testing scrape_reddit...")
+try:
+    reddit_tool = tools.get("scrape_reddit")
+    if reddit_tool:
+        result = reddit_tool.invoke({"keywords": ["sri lanka"], "limit": 3})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if "error" in parsed:
+            print(f"[WARN] Reddit returned error: {parsed['error']}")
+        else:
+            count = len(parsed.get('results', parsed.get('posts', [])))
+            print(f"[OK] Reddit: {count} posts fetched")
+    else:
+        print("[WARN] scrape_reddit not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] Reddit: {e}")
+
+print("\n" + "="*60)
+print("[DONE] News & official source tools test complete")
+print("="*60)

test_social_tools.py

@@ -0,0 +1,185 @@
+"""
+test_social_tools.py
+Test for social media tools (session required)
+Will identify which sessions need to be created/refreshed
+"""
+import sys
+import os
+import io
+import json
+
+# Force UTF-8 output
+if sys.platform == 'win32':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+print("="*60)
+print("[TEST] SOCIAL MEDIA TOOLS (Session Required)")
+print("="*60)
+
+# Check for existing sessions
+print("\n[SESSIONS] Checking for existing session files...")
+session_dirs = [
+    "src/utils/.sessions",
+    ".sessions",
+]
+
+session_files = {}
+for dir in session_dirs:
+    if os.path.exists(dir):
+        for f in os.listdir(dir):
+            if f.endswith('.json'):
+                session_files[f] = os.path.join(dir, f)
+
+if session_files:
+    print(f"[OK] Found {len(session_files)} session files:")
+    for name, path in session_files.items():
+        print(f"   - {name}: {path}")
+else:
+    print("[WARN] No session files found!")
+    print("   Please create sessions using the session manager")
+
+# Try to create ToolSet with profile scrapers
+print("\n[TEST] Creating ToolSet with profile scrapers...")
+try:
+    from src.utils.tool_factory import create_tool_set
+    tools = create_tool_set(include_profile_scrapers=True)
+    print("[OK] ToolSet created")
+except Exception as e:
+    print(f"[FAIL] Could not create ToolSet: {e}")
+    sys.exit(1)
+
+# Test Twitter
+print("\n[1] Testing scrape_twitter (keyword search)...")
+try:
+    twitter_tool = tools.get("scrape_twitter")
+    if twitter_tool:
+        result = twitter_tool.invoke({"query": "sri lanka", "max_items": 2})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if isinstance(parsed, dict) and "error" in parsed:
+            if "session" in str(parsed['error']).lower():
+                print(f"[SESSION] Twitter: Session not found or expired")
+                print(f"   Error: {parsed['error'][:100]}")
+            else:
+                print(f"[WARN] Twitter error: {parsed['error'][:100]}")
+        elif isinstance(parsed, dict):
+            count = len(parsed.get('results', []))
+            print(f"[OK] Twitter: {count} tweets fetched")
+        elif isinstance(parsed, list):
+            print(f"[OK] Twitter: {len(parsed)} tweets fetched")
+        else:
+            print(f"[OK] Twitter returned: {type(parsed)}")
+    else:
+        print("[WARN] scrape_twitter not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] Twitter: {e}")
+
+# Test Facebook
+print("\n[2] Testing scrape_facebook (keyword search)...")
+try:
+    fb_tool = tools.get("scrape_facebook")
+    if fb_tool:
+        result = fb_tool.invoke({"keywords": ["sri lanka"], "max_items": 2})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if isinstance(parsed, dict) and "error" in parsed:
+            if "session" in str(parsed['error']).lower():
+                print(f"[SESSION] Facebook: Session not found or expired")
+                print(f"   Error: {parsed['error'][:100]}")
+            else:
+                print(f"[WARN] Facebook error: {parsed['error'][:100]}")
+        elif isinstance(parsed, dict):
+            count = len(parsed.get('results', []))
+            print(f"[OK] Facebook: {count} posts fetched")
+        elif isinstance(parsed, list):
+            print(f"[OK] Facebook: {len(parsed)} posts fetched")
+        else:
+            print(f"[OK] Facebook returned: {type(parsed)}")
+    else:
+        print("[WARN] scrape_facebook not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] Facebook: {e}")
+
+# Test LinkedIn
+print("\n[3] Testing scrape_linkedin (keyword search)...")
+try:
+    linkedin_tool = tools.get("scrape_linkedin")
+    if linkedin_tool:
+        result = linkedin_tool.invoke({"keywords": ["sri lanka"], "max_items": 2})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if isinstance(parsed, dict) and "error" in parsed:
+            if "session" in str(parsed['error']).lower():
+                print(f"[SESSION] LinkedIn: Session not found or expired")
+                print(f"   Error: {parsed['error'][:100]}")
+            else:
+                print(f"[WARN] LinkedIn error: {parsed['error'][:100]}")
+        elif isinstance(parsed, dict):
+            count = len(parsed.get('results', []))
+            print(f"[OK] LinkedIn: {count} posts fetched")
+        elif isinstance(parsed, list):
+            print(f"[OK] LinkedIn: {len(parsed)} posts fetched")
+        else:
+            print(f"[OK] LinkedIn returned: {type(parsed)}")
+    else:
+        print("[WARN] scrape_linkedin not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] LinkedIn: {e}")
+
+# Test Instagram
+print("\n[4] Testing scrape_instagram (hashtag search)...")
+try:
+    instagram_tool = tools.get("scrape_instagram")
+    if instagram_tool:
+        result = instagram_tool.invoke({"keywords": ["srilanka"], "max_items": 2})
+        parsed = json.loads(result) if isinstance(result, str) else result
+        if isinstance(parsed, dict) and "error" in parsed:
+            if "session" in str(parsed['error']).lower():
+                print(f"[SESSION] Instagram: Session not found or expired")
+                print(f"   Error: {parsed['error'][:100]}")
+            else:
+                print(f"[WARN] Instagram error: {parsed['error'][:100]}")
+        elif isinstance(parsed, dict):
+            count = len(parsed.get('results', []))
+            print(f"[OK] Instagram: {count} posts fetched")
+        elif isinstance(parsed, list):
+            print(f"[OK] Instagram: {len(parsed)} posts fetched")
+        else:
+            print(f"[OK] Instagram returned: {type(parsed)}")
+    else:
+        print("[WARN] scrape_instagram not found in ToolSet")
+except Exception as e:
+    print(f"[FAIL] Instagram: {e}")
+
+# Test Profile Scrapers
+print("\n[5] Testing scrape_twitter_profile (specific account)...")
+try:
+    from src.utils.profile_scrapers import scrape_twitter_profile
+    result = scrape_twitter_profile.invoke({"username": "SLTMobitel", "max_items": 2})
+    parsed = json.loads(result) if isinstance(result, str) else result
+    if isinstance(parsed, dict) and "error" in parsed:
+        if "session" in str(parsed['error']).lower():
+            print(f"[SESSION] Twitter Profile: Session not found or expired")
+            print(f"   Error: {parsed['error'][:100]}")
+        elif "timeout" in str(parsed['error']).lower():
+            print(f"[TIMEOUT] Twitter Profile: Navigation timed out (X blocks automation)")
+            print(f"   Error: {parsed['error'][:100]}")
+        else:
+            print(f"[WARN] Twitter Profile error: {parsed['error'][:100]}")
+    elif isinstance(parsed, dict):
+        count = len(parsed.get('results', []))
+        print(f"[OK] Twitter Profile: {count} tweets fetched from @SLTMobitel")
+    else:
+        print(f"[OK] Twitter Profile returned: {type(parsed)}")
+except Exception as e:
+    print(f"[FAIL] Twitter Profile: {e}")
+
+print("\n" + "="*60)
+print("[SUMMARY]")
+print("="*60)
+print("If you see [SESSION] errors, please create new sessions using:")
+print("  - Twitter: Run session manager with Twitter login")
+print("  - Facebook: Run session manager with Facebook login") 
+print("  - LinkedIn: Run session manager with LinkedIn login")
+print("  - Instagram: Run session manager with Instagram login")
+print("\nSession manager: python src/utils/session_manager.py")
+print("="*60)

test_weather_tools.py

@@ -0,0 +1,57 @@
+"""
+test_weather_tools.py
+Quick test for weather and flood tools only (no sessions required)
+"""
+import sys
+import os
+import io
+
+# Force UTF-8 output
+if sys.platform == 'win32':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+print("="*60)
+print("[TEST] WEATHER & FLOOD TOOLS")
+print("="*60)
+
+# Test 1: DMC Alerts
+print("\n[1] Testing tool_dmc_alerts...")
+try:
+    from src.utils.utils import tool_dmc_alerts
+    result = tool_dmc_alerts()
+    print(f"[OK] DMC Alerts: {str(result)[:300]}...")
+except Exception as e:
+    print(f"[FAIL] DMC Alerts: {e}")
+
+# Test 2: Weather Nowcast
+print("\n[2] Testing tool_weather_nowcast...")
+try:
+    from src.utils.utils import tool_weather_nowcast
+    result = tool_weather_nowcast()
+    print(f"[OK] Weather Nowcast: {str(result)[:300]}...")
+except Exception as e:
+    print(f"[FAIL] Weather Nowcast: {e}")
+
+# Test 3: District Weather
+print("\n[3] Testing tool_district_weather...")
+try:
+    from src.utils.utils import tool_district_weather
+    result = tool_district_weather("colombo")
+    print(f"[OK] District Weather: {str(result)[:300]}...")
+except Exception as e:
+    print(f"[FAIL] District Weather: {e}")
+
+# Test 4: RiverNet (may take longer - uses Playwright)
+print("\n[4] Testing tool_rivernet_status (may take 30-60 seconds)...")
+try:
+    from src.utils.utils import tool_rivernet_status
+    result = tool_rivernet_status()
+    print(f"[OK] RiverNet: {str(result)[:300]}...")
+except Exception as e:
+    print(f"[FAIL] RiverNet: {e}")
+
+print("\n" + "="*60)
+print("[DONE] Weather tools test complete")
+print("="*60)

tool_test_results.json

@@ -0,0 +1,73 @@
+{
+  "timestamp": "2025-12-08T14:41:26.602415",
+  "summary": {
+    "success": 14,
+    "errors": 1,
+    "session_issues": [],
+    "other_errors": [
+      "scrape_twitter_profile"
+    ]
+  },
+  "details": {
+    "tool_dmc_alerts": {
+      "status": "success",
+      "error": null
+    },
+    "tool_weather_nowcast": {
+      "status": "success",
+      "error": null
+    },
+    "tool_rivernet_status": {
+      "status": "success",
+      "error": null
+    },
+    "tool_district_weather": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_local_news": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_cse_stock_data": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_government_gazette": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_parliament_minutes": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_reddit": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_twitter": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_facebook": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_linkedin": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_instagram": {
+      "status": "success",
+      "error": null
+    },
+    "scrape_twitter_profile": {
+      "status": "error",
+      "error": "Profile not found or private: @SLTMobitel"
+    },
+    "scrape_facebook_profile": {
+      "status": "success",
+      "error": null
+    }
+  }
+}