Enhanced system initialization, context awareness, and debug information
Browse files- app.py +60 -15
- core/coordinator.py +0 -53
- core/providers/huggingface.py +47 -24
- core/providers/ollama.py +46 -16
- services/hf_endpoint_monitor.py +35 -55
- services/weather.py +14 -28
app.py
CHANGED
|
@@ -15,6 +15,7 @@ from core.coordinator import coordinator
|
|
| 15 |
from core.errors import translate_error
|
| 16 |
from core.personality import personality
|
| 17 |
from services.hf_endpoint_monitor import hf_monitor
|
|
|
|
| 18 |
import logging
|
| 19 |
|
| 20 |
# Set up logging
|
|
@@ -116,15 +117,24 @@ with st.sidebar:
|
|
| 116 |
|
| 117 |
try:
|
| 118 |
hf_status = hf_monitor.check_endpoint_status()
|
| 119 |
-
|
|
|
|
| 120 |
if hf_status.get('initialized', False):
|
| 121 |
-
st.success("🤗 HF: Available
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
else:
|
| 123 |
-
st.warning("
|
|
|
|
|
|
|
| 124 |
else:
|
| 125 |
-
st.
|
| 126 |
-
except:
|
| 127 |
-
st.info("
|
| 128 |
|
| 129 |
if check_redis_health():
|
| 130 |
st.success("💾 Redis: Connected")
|
|
@@ -134,20 +144,57 @@ with st.sidebar:
|
|
| 134 |
st.divider()
|
| 135 |
|
| 136 |
st.subheader("🐛 Debug Info")
|
| 137 |
-
# Show
|
| 138 |
-
st.markdown(f"Environment
|
| 139 |
-
st.markdown(f"Model
|
| 140 |
-
st.markdown(f"
|
| 141 |
|
| 142 |
# Show active features
|
| 143 |
features = []
|
| 144 |
-
if config.hf_token:
|
| 145 |
-
features.append("HF Expert")
|
| 146 |
if os.getenv("TAVILY_API_KEY"):
|
| 147 |
features.append("Web Search")
|
| 148 |
if config.openweather_api_key:
|
| 149 |
features.append("Weather")
|
| 150 |
-
st.markdown(f"Active Features
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
# Main interface
|
| 153 |
st.title("🐱 CosmicCat AI Assistant")
|
|
@@ -543,8 +590,6 @@ with tab2:
|
|
| 543 |
features = []
|
| 544 |
if config.use_fallback:
|
| 545 |
features.append("Fallback Mode")
|
| 546 |
-
if config.hf_token:
|
| 547 |
-
features.append("HF Deep Analysis")
|
| 548 |
if os.getenv("TAVILY_API_KEY"):
|
| 549 |
features.append("Web Search")
|
| 550 |
if config.openweather_api_key:
|
|
|
|
| 15 |
from core.errors import translate_error
|
| 16 |
from core.personality import personality
|
| 17 |
from services.hf_endpoint_monitor import hf_monitor
|
| 18 |
+
from services.weather import weather_service
|
| 19 |
import logging
|
| 20 |
|
| 21 |
# Set up logging
|
|
|
|
| 117 |
|
| 118 |
try:
|
| 119 |
hf_status = hf_monitor.check_endpoint_status()
|
| 120 |
+
# Enhanced HF status display
|
| 121 |
+
if hf_status.get('available'):
|
| 122 |
if hf_status.get('initialized', False):
|
| 123 |
+
st.success(f"🤗 HF Endpoint: Available ({hf_status.get('status_code')} OK)")
|
| 124 |
+
if hf_status.get('model'):
|
| 125 |
+
st.info(f" Model: {hf_status.get('model')}")
|
| 126 |
+
if hf_status.get('region'):
|
| 127 |
+
st.info(f" Region: {hf_status.get('region')}")
|
| 128 |
+
if hf_status.get('warmup_count'):
|
| 129 |
+
st.info(f" Warmup Count: {hf_status.get('warmup_count')}")
|
| 130 |
else:
|
| 131 |
+
st.warning("⏳ Initializing...")
|
| 132 |
+
elif hf_status.get('status_code') == 200:
|
| 133 |
+
st.info("📡 Connecting...")
|
| 134 |
else:
|
| 135 |
+
st.error("🔴 Unavailable")
|
| 136 |
+
except Exception as e:
|
| 137 |
+
st.info("⏳ Initializing...")
|
| 138 |
|
| 139 |
if check_redis_health():
|
| 140 |
st.success("💾 Redis: Connected")
|
|
|
|
| 144 |
st.divider()
|
| 145 |
|
| 146 |
st.subheader("🐛 Debug Info")
|
| 147 |
+
# Show enhanced debug information
|
| 148 |
+
st.markdown(f"**Environment:** {'HF Space' if config.is_hf_space else 'Local'}")
|
| 149 |
+
st.markdown(f"**Model:** {st.session_state.selected_model}")
|
| 150 |
+
st.markdown(f"**Fallback:** {'Enabled' if config.use_fallback else 'Disabled'}")
|
| 151 |
|
| 152 |
# Show active features
|
| 153 |
features = []
|
|
|
|
|
|
|
| 154 |
if os.getenv("TAVILY_API_KEY"):
|
| 155 |
features.append("Web Search")
|
| 156 |
if config.openweather_api_key:
|
| 157 |
features.append("Weather")
|
| 158 |
+
st.markdown(f"**Active Features:** {', '.join(features) if features else 'None'}")
|
| 159 |
+
|
| 160 |
+
# Show recent activity
|
| 161 |
+
try:
|
| 162 |
+
user_session = session_manager.get_session("default_user")
|
| 163 |
+
coord_stats = user_session.get('ai_coordination', {})
|
| 164 |
+
if coord_stats and coord_stats.get('last_coordination'):
|
| 165 |
+
st.markdown(f"**Last Request:** {coord_stats.get('last_coordination')}")
|
| 166 |
+
else:
|
| 167 |
+
st.markdown("**Last Request:** N/A")
|
| 168 |
+
except:
|
| 169 |
+
st.markdown("**Last Request:** N/A")
|
| 170 |
+
|
| 171 |
+
# Show Ollama ping status
|
| 172 |
+
try:
|
| 173 |
+
import requests
|
| 174 |
+
import time
|
| 175 |
+
start_time = time.time()
|
| 176 |
+
headers = {
|
| 177 |
+
"ngrok-skip-browser-warning": "true",
|
| 178 |
+
"User-Agent": "CosmicCat-Debug"
|
| 179 |
+
}
|
| 180 |
+
response = requests.get(
|
| 181 |
+
f"{st.session_state.ngrok_url_temp}/api/tags",
|
| 182 |
+
headers=headers,
|
| 183 |
+
timeout=15
|
| 184 |
+
)
|
| 185 |
+
ping_time = round((time.time() - start_time) * 1000)
|
| 186 |
+
if response.status_code == 200:
|
| 187 |
+
st.markdown(f"**Ollama Ping:** {response.status_code} OK ({ping_time}ms)")
|
| 188 |
+
else:
|
| 189 |
+
st.markdown(f"**Ollama Ping:** {response.status_code} Error")
|
| 190 |
+
except Exception as e:
|
| 191 |
+
st.markdown("**Ollama Ping:** Unreachable")
|
| 192 |
+
|
| 193 |
+
# Redis status
|
| 194 |
+
if check_redis_health():
|
| 195 |
+
st.markdown("**Redis:** Healthy")
|
| 196 |
+
else:
|
| 197 |
+
st.markdown("**Redis:** Unhealthy")
|
| 198 |
|
| 199 |
# Main interface
|
| 200 |
st.title("🐱 CosmicCat AI Assistant")
|
|
|
|
| 590 |
features = []
|
| 591 |
if config.use_fallback:
|
| 592 |
features.append("Fallback Mode")
|
|
|
|
|
|
|
| 593 |
if os.getenv("TAVILY_API_KEY"):
|
| 594 |
features.append("Web Search")
|
| 595 |
if config.openweather_api_key:
|
core/coordinator.py
CHANGED
|
@@ -72,59 +72,6 @@ class AICoordinator:
|
|
| 72 |
"reasoning": f"Found topics requiring current info: {', '.join(search_topics)}" if search_topics else "No current info needed"
|
| 73 |
}
|
| 74 |
|
| 75 |
-
def manual_hf_analysis(self, user_id: str, conversation_history: List[Dict]) -> str:
|
| 76 |
-
"""Perform manual HF analysis with web search integration"""
|
| 77 |
-
try:
|
| 78 |
-
# Determine research needs
|
| 79 |
-
research_decision = self.determine_web_search_needs(conversation_history)
|
| 80 |
-
|
| 81 |
-
# Prepare enhanced prompt for HF
|
| 82 |
-
system_prompt = f"""
|
| 83 |
-
You are a deep analysis expert joining an ongoing conversation.
|
| 84 |
-
Research Decision: {research_decision['reasoning']}
|
| 85 |
-
Please provide:
|
| 86 |
-
1. Deep insights on conversation themes
|
| 87 |
-
2. Research/web search needs (if any)
|
| 88 |
-
3. Strategic recommendations
|
| 89 |
-
4. Questions to explore further
|
| 90 |
-
Conversation History:
|
| 91 |
-
"""
|
| 92 |
-
|
| 93 |
-
# Add conversation history to messages
|
| 94 |
-
messages = [{"role": "system", "content": system_prompt}]
|
| 95 |
-
|
| 96 |
-
# Add recent conversation (last 15 messages for context)
|
| 97 |
-
for msg in conversation_history[-15:]: # Ensure all messages have proper format
|
| 98 |
-
if isinstance(msg, dict) and "role" in msg and "content" in msg:
|
| 99 |
-
messages.append({
|
| 100 |
-
"role": msg["role"],
|
| 101 |
-
"content": msg["content"]
|
| 102 |
-
})
|
| 103 |
-
|
| 104 |
-
# Get HF provider
|
| 105 |
-
from core.llm_factory import llm_factory
|
| 106 |
-
hf_provider = llm_factory.get_provider('huggingface')
|
| 107 |
-
|
| 108 |
-
if hf_provider:
|
| 109 |
-
# Generate deep analysis with full 8192 token capacity
|
| 110 |
-
response = hf_provider.generate("Deep analysis request", messages)
|
| 111 |
-
return response or "HF Expert analysis completed."
|
| 112 |
-
else:
|
| 113 |
-
return "❌ HF provider not available."
|
| 114 |
-
|
| 115 |
-
except Exception as e:
|
| 116 |
-
return f"❌ HF analysis failed: {str(e)}"
|
| 117 |
-
|
| 118 |
-
# Add this method to show HF engagement status
|
| 119 |
-
def get_hf_engagement_status(self) -> Dict:
|
| 120 |
-
"""Get current HF engagement status"""
|
| 121 |
-
return {
|
| 122 |
-
"hf_available": self._check_hf_availability(),
|
| 123 |
-
"web_search_configured": bool(self.tavily_client),
|
| 124 |
-
"research_needs_detected": False, # Will be determined per conversation
|
| 125 |
-
"last_hf_analysis": None # Track last analysis time
|
| 126 |
-
}
|
| 127 |
-
|
| 128 |
async def coordinate_cosmic_response(self, user_id: str, user_query: str) -> AsyncGenerator[Dict, None]:
|
| 129 |
"""
|
| 130 |
Three-stage cosmic response cascade:
|
|
|
|
| 72 |
"reasoning": f"Found topics requiring current info: {', '.join(search_topics)}" if search_topics else "No current info needed"
|
| 73 |
}
|
| 74 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
async def coordinate_cosmic_response(self, user_id: str, user_query: str) -> AsyncGenerator[Dict, None]:
|
| 76 |
"""
|
| 77 |
Three-stage cosmic response cascade:
|
core/providers/huggingface.py
CHANGED
|
@@ -4,7 +4,9 @@ from datetime import datetime
|
|
| 4 |
from typing import List, Dict, Optional, Union
|
| 5 |
from core.providers.base import LLMProvider
|
| 6 |
from utils.config import config
|
|
|
|
| 7 |
logger = logging.getLogger(__name__)
|
|
|
|
| 8 |
try:
|
| 9 |
from openai import OpenAI
|
| 10 |
HUGGINGFACE_SDK_AVAILABLE = True
|
|
@@ -14,18 +16,19 @@ except ImportError:
|
|
| 14 |
|
| 15 |
class HuggingFaceProvider(LLMProvider):
|
| 16 |
"""Hugging Face LLM provider implementation"""
|
| 17 |
-
|
| 18 |
def __init__(self, model_name: str, timeout: int = 30, max_retries: int = 3):
|
| 19 |
super().__init__(model_name, timeout, max_retries)
|
| 20 |
logger.info(f"Initializing HuggingFaceProvider with:")
|
| 21 |
logger.info(f" HF_API_URL: {config.hf_api_url}")
|
| 22 |
logger.info(f" HF_TOKEN SET: {bool(config.hf_token)}")
|
| 23 |
-
|
| 24 |
if not HUGGINGFACE_SDK_AVAILABLE:
|
| 25 |
raise ImportError("Hugging Face provider requires 'openai' package")
|
|
|
|
| 26 |
if not config.hf_token:
|
| 27 |
raise ValueError("HF_TOKEN not set - required for Hugging Face provider")
|
| 28 |
-
|
| 29 |
# Make sure NO proxies parameter is included
|
| 30 |
try:
|
| 31 |
self.client = OpenAI(
|
|
@@ -37,7 +40,7 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 37 |
logger.error(f"Failed to initialize HuggingFaceProvider: {e}")
|
| 38 |
logger.error(f"Error type: {type(e)}")
|
| 39 |
raise
|
| 40 |
-
|
| 41 |
def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
|
| 42 |
"""Generate a response synchronously"""
|
| 43 |
try:
|
|
@@ -45,7 +48,7 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 45 |
except Exception as e:
|
| 46 |
logger.error(f"Hugging Face generation failed: {e}")
|
| 47 |
return None
|
| 48 |
-
|
| 49 |
def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
|
| 50 |
"""Generate a response with streaming support"""
|
| 51 |
try:
|
|
@@ -53,7 +56,7 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 53 |
except Exception as e:
|
| 54 |
logger.error(f"Hugging Face stream generation failed: {e}")
|
| 55 |
return None
|
| 56 |
-
|
| 57 |
def validate_model(self) -> bool:
|
| 58 |
"""Validate if the model is available"""
|
| 59 |
# For Hugging Face endpoints, we'll assume the model is valid if we can connect
|
|
@@ -64,14 +67,18 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 64 |
except Exception as e:
|
| 65 |
logger.warning(f"Hugging Face model validation failed: {e}")
|
| 66 |
return False
|
| 67 |
-
|
| 68 |
def _generate_impl(self, prompt: str, conversation_history: List[Dict]) -> str:
|
| 69 |
-
"""Implementation of synchronous generation with proper configuration"""
|
| 70 |
-
# Inject
|
| 71 |
current_time = datetime.now().strftime("%A, %B %d, %Y at %I:%M %p")
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
try:
|
| 76 |
response = self.client.chat.completions.create(
|
| 77 |
model=self.model_name,
|
|
@@ -88,6 +95,7 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 88 |
if self._is_scale_to_zero_error(e):
|
| 89 |
logger.info("Hugging Face endpoint is scaling up, waiting...")
|
| 90 |
time.sleep(60) # Wait for endpoint to initialize
|
|
|
|
| 91 |
# Retry once after waiting
|
| 92 |
response = self.client.chat.completions.create(
|
| 93 |
model=self.model_name,
|
|
@@ -101,14 +109,18 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 101 |
return response.choices[0].message.content
|
| 102 |
else:
|
| 103 |
raise
|
| 104 |
-
|
| 105 |
def _stream_generate_impl(self, prompt: str, conversation_history: List[Dict]) -> List[str]:
|
| 106 |
-
"""Implementation of streaming generation with proper configuration"""
|
| 107 |
-
# Inject
|
| 108 |
current_time = datetime.now().strftime("%A, %B %d, %Y at %I:%M %p")
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
try:
|
| 113 |
response = self.client.chat.completions.create(
|
| 114 |
model=self.model_name,
|
|
@@ -120,19 +132,18 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 120 |
presence_penalty=0.1,
|
| 121 |
stream=True # Enable streaming
|
| 122 |
)
|
| 123 |
-
|
| 124 |
chunks = []
|
| 125 |
for chunk in response:
|
| 126 |
content = chunk.choices[0].delta.content
|
| 127 |
if content:
|
| 128 |
chunks.append(content)
|
| 129 |
-
|
| 130 |
return chunks
|
| 131 |
except Exception as e:
|
| 132 |
# Handle scale-to-zero behavior
|
| 133 |
if self._is_scale_to_zero_error(e):
|
| 134 |
logger.info("Hugging Face endpoint is scaling up, waiting...")
|
| 135 |
time.sleep(60) # Wait for endpoint to initialize
|
|
|
|
| 136 |
# Retry once after waiting
|
| 137 |
response = self.client.chat.completions.create(
|
| 138 |
model=self.model_name,
|
|
@@ -144,21 +155,33 @@ class HuggingFaceProvider(LLMProvider):
|
|
| 144 |
presence_penalty=0.1,
|
| 145 |
stream=True # Enable streaming
|
| 146 |
)
|
| 147 |
-
|
| 148 |
chunks = []
|
| 149 |
for chunk in response:
|
| 150 |
content = chunk.choices[0].delta.content
|
| 151 |
if content:
|
| 152 |
chunks.append(content)
|
| 153 |
-
|
| 154 |
return chunks
|
| 155 |
else:
|
| 156 |
raise
|
| 157 |
-
|
| 158 |
def _is_scale_to_zero_error(self, error: Exception) -> bool:
|
| 159 |
"""Check if the error is related to scale-to-zero initialization"""
|
| 160 |
error_str = str(error).lower()
|
| 161 |
scale_to_zero_indicators = [
|
| 162 |
-
"503",
|
|
|
|
|
|
|
|
|
|
| 163 |
]
|
| 164 |
return any(indicator in error_str for indicator in scale_to_zero_indicators)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
from typing import List, Dict, Optional, Union
|
| 5 |
from core.providers.base import LLMProvider
|
| 6 |
from utils.config import config
|
| 7 |
+
from services.weather import weather_service
|
| 8 |
logger = logging.getLogger(__name__)
|
| 9 |
+
|
| 10 |
try:
|
| 11 |
from openai import OpenAI
|
| 12 |
HUGGINGFACE_SDK_AVAILABLE = True
|
|
|
|
| 16 |
|
| 17 |
class HuggingFaceProvider(LLMProvider):
|
| 18 |
"""Hugging Face LLM provider implementation"""
|
| 19 |
+
|
| 20 |
def __init__(self, model_name: str, timeout: int = 30, max_retries: int = 3):
|
| 21 |
super().__init__(model_name, timeout, max_retries)
|
| 22 |
logger.info(f"Initializing HuggingFaceProvider with:")
|
| 23 |
logger.info(f" HF_API_URL: {config.hf_api_url}")
|
| 24 |
logger.info(f" HF_TOKEN SET: {bool(config.hf_token)}")
|
| 25 |
+
|
| 26 |
if not HUGGINGFACE_SDK_AVAILABLE:
|
| 27 |
raise ImportError("Hugging Face provider requires 'openai' package")
|
| 28 |
+
|
| 29 |
if not config.hf_token:
|
| 30 |
raise ValueError("HF_TOKEN not set - required for Hugging Face provider")
|
| 31 |
+
|
| 32 |
# Make sure NO proxies parameter is included
|
| 33 |
try:
|
| 34 |
self.client = OpenAI(
|
|
|
|
| 40 |
logger.error(f"Failed to initialize HuggingFaceProvider: {e}")
|
| 41 |
logger.error(f"Error type: {type(e)}")
|
| 42 |
raise
|
| 43 |
+
|
| 44 |
def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
|
| 45 |
"""Generate a response synchronously"""
|
| 46 |
try:
|
|
|
|
| 48 |
except Exception as e:
|
| 49 |
logger.error(f"Hugging Face generation failed: {e}")
|
| 50 |
return None
|
| 51 |
+
|
| 52 |
def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
|
| 53 |
"""Generate a response with streaming support"""
|
| 54 |
try:
|
|
|
|
| 56 |
except Exception as e:
|
| 57 |
logger.error(f"Hugging Face stream generation failed: {e}")
|
| 58 |
return None
|
| 59 |
+
|
| 60 |
def validate_model(self) -> bool:
|
| 61 |
"""Validate if the model is available"""
|
| 62 |
# For Hugging Face endpoints, we'll assume the model is valid if we can connect
|
|
|
|
| 67 |
except Exception as e:
|
| 68 |
logger.warning(f"Hugging Face model validation failed: {e}")
|
| 69 |
return False
|
| 70 |
+
|
| 71 |
def _generate_impl(self, prompt: str, conversation_history: List[Dict]) -> str:
|
| 72 |
+
"""Implementation of synchronous generation with proper configuration and context injection"""
|
| 73 |
+
# Inject context message with current time/date/weather
|
| 74 |
current_time = datetime.now().strftime("%A, %B %d, %Y at %I:%M %p")
|
| 75 |
+
weather_summary = self._get_weather_summary()
|
| 76 |
+
context_msg = {
|
| 77 |
+
"role": "system",
|
| 78 |
+
"content": f"[Current Context: {current_time} | Weather: {weather_summary}]"
|
| 79 |
+
}
|
| 80 |
+
enhanced_history = [context_msg] + conversation_history
|
| 81 |
+
|
| 82 |
try:
|
| 83 |
response = self.client.chat.completions.create(
|
| 84 |
model=self.model_name,
|
|
|
|
| 95 |
if self._is_scale_to_zero_error(e):
|
| 96 |
logger.info("Hugging Face endpoint is scaling up, waiting...")
|
| 97 |
time.sleep(60) # Wait for endpoint to initialize
|
| 98 |
+
|
| 99 |
# Retry once after waiting
|
| 100 |
response = self.client.chat.completions.create(
|
| 101 |
model=self.model_name,
|
|
|
|
| 109 |
return response.choices[0].message.content
|
| 110 |
else:
|
| 111 |
raise
|
| 112 |
+
|
| 113 |
def _stream_generate_impl(self, prompt: str, conversation_history: List[Dict]) -> List[str]:
|
| 114 |
+
"""Implementation of streaming generation with proper configuration and context injection"""
|
| 115 |
+
# Inject context message with current time/date/weather
|
| 116 |
current_time = datetime.now().strftime("%A, %B %d, %Y at %I:%M %p")
|
| 117 |
+
weather_summary = self._get_weather_summary()
|
| 118 |
+
context_msg = {
|
| 119 |
+
"role": "system",
|
| 120 |
+
"content": f"[Current Context: {current_time} | Weather: {weather_summary}]"
|
| 121 |
+
}
|
| 122 |
+
enhanced_history = [context_msg] + conversation_history
|
| 123 |
+
|
| 124 |
try:
|
| 125 |
response = self.client.chat.completions.create(
|
| 126 |
model=self.model_name,
|
|
|
|
| 132 |
presence_penalty=0.1,
|
| 133 |
stream=True # Enable streaming
|
| 134 |
)
|
|
|
|
| 135 |
chunks = []
|
| 136 |
for chunk in response:
|
| 137 |
content = chunk.choices[0].delta.content
|
| 138 |
if content:
|
| 139 |
chunks.append(content)
|
|
|
|
| 140 |
return chunks
|
| 141 |
except Exception as e:
|
| 142 |
# Handle scale-to-zero behavior
|
| 143 |
if self._is_scale_to_zero_error(e):
|
| 144 |
logger.info("Hugging Face endpoint is scaling up, waiting...")
|
| 145 |
time.sleep(60) # Wait for endpoint to initialize
|
| 146 |
+
|
| 147 |
# Retry once after waiting
|
| 148 |
response = self.client.chat.completions.create(
|
| 149 |
model=self.model_name,
|
|
|
|
| 155 |
presence_penalty=0.1,
|
| 156 |
stream=True # Enable streaming
|
| 157 |
)
|
|
|
|
| 158 |
chunks = []
|
| 159 |
for chunk in response:
|
| 160 |
content = chunk.choices[0].delta.content
|
| 161 |
if content:
|
| 162 |
chunks.append(content)
|
|
|
|
| 163 |
return chunks
|
| 164 |
else:
|
| 165 |
raise
|
| 166 |
+
|
| 167 |
def _is_scale_to_zero_error(self, error: Exception) -> bool:
|
| 168 |
"""Check if the error is related to scale-to-zero initialization"""
|
| 169 |
error_str = str(error).lower()
|
| 170 |
scale_to_zero_indicators = [
|
| 171 |
+
"503",
|
| 172 |
+
"service unavailable",
|
| 173 |
+
"initializing",
|
| 174 |
+
"cold start"
|
| 175 |
]
|
| 176 |
return any(indicator in error_str for indicator in scale_to_zero_indicators)
|
| 177 |
+
|
| 178 |
+
def _get_weather_summary(self) -> str:
|
| 179 |
+
"""Get formatted weather summary"""
|
| 180 |
+
try:
|
| 181 |
+
weather = weather_service.get_current_weather("New York")
|
| 182 |
+
if weather:
|
| 183 |
+
return f"{weather.get('temperature', 'N/A')}°C, {weather.get('description', 'Clear skies')}"
|
| 184 |
+
else:
|
| 185 |
+
return "Clear skies"
|
| 186 |
+
except:
|
| 187 |
+
return "Clear skies"
|
core/providers/ollama.py
CHANGED
|
@@ -5,14 +5,15 @@ from datetime import datetime
|
|
| 5 |
from typing import List, Dict, Optional, Union
|
| 6 |
from core.providers.base import LLMProvider
|
| 7 |
from utils.config import config
|
| 8 |
-
from
|
| 9 |
|
| 10 |
logger = logging.getLogger(__name__)
|
| 11 |
|
| 12 |
class OllamaProvider(LLMProvider):
|
| 13 |
"""Ollama LLM provider implementation"""
|
| 14 |
|
| 15 |
-
def __init__(self, model_name: str, timeout: int = 60, max_retries: int = 3):
|
|
|
|
| 16 |
super().__init__(model_name, timeout, max_retries)
|
| 17 |
self.host = self._sanitize_host(config.ollama_host or "http://localhost:11434")
|
| 18 |
# Headers to skip ngrok browser warning
|
|
@@ -20,7 +21,7 @@ class OllamaProvider(LLMProvider):
|
|
| 20 |
"ngrok-skip-browser-warning": "true",
|
| 21 |
"User-Agent": "CosmicCat-AI-Assistant"
|
| 22 |
}
|
| 23 |
-
|
| 24 |
def _sanitize_host(self, host: str) -> str:
|
| 25 |
"""Sanitize host URL by removing whitespace and control characters"""
|
| 26 |
if not host:
|
|
@@ -33,7 +34,7 @@ class OllamaProvider(LLMProvider):
|
|
| 33 |
if not host.startswith(('http://', 'https://')):
|
| 34 |
host = 'http://' + host
|
| 35 |
return host
|
| 36 |
-
|
| 37 |
def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
|
| 38 |
"""Generate a response synchronously"""
|
| 39 |
try:
|
|
@@ -41,7 +42,7 @@ class OllamaProvider(LLMProvider):
|
|
| 41 |
except Exception as e:
|
| 42 |
logger.error(f"Ollama generation failed: {e}")
|
| 43 |
return None
|
| 44 |
-
|
| 45 |
def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
|
| 46 |
"""Generate a response with streaming support"""
|
| 47 |
try:
|
|
@@ -49,7 +50,7 @@ class OllamaProvider(LLMProvider):
|
|
| 49 |
except Exception as e:
|
| 50 |
logger.error(f"Ollama stream generation failed: {e}")
|
| 51 |
return None
|
| 52 |
-
|
| 53 |
def validate_model(self) -> bool:
|
| 54 |
"""Validate if the model is available"""
|
| 55 |
try:
|
|
@@ -74,19 +75,28 @@ class OllamaProvider(LLMProvider):
|
|
| 74 |
except Exception as e:
|
| 75 |
logger.error(f"Model validation failed: {e}")
|
| 76 |
return False
|
| 77 |
-
|
| 78 |
def _generate_impl(self, prompt: str, conversation_history: List[Dict]) -> str:
|
| 79 |
-
"""Implementation of synchronous generation"""
|
| 80 |
url = f"{self.host}/api/chat"
|
| 81 |
messages = conversation_history.copy()
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
# Add the current prompt if not already in history
|
| 84 |
if not messages or messages[-1].get("content") != prompt:
|
| 85 |
-
|
| 86 |
-
|
| 87 |
payload = {
|
| 88 |
"model": self.model_name,
|
| 89 |
-
"messages":
|
| 90 |
"stream": False
|
| 91 |
}
|
| 92 |
|
|
@@ -99,19 +109,28 @@ class OllamaProvider(LLMProvider):
|
|
| 99 |
response.raise_for_status()
|
| 100 |
result = response.json()
|
| 101 |
return result["message"]["content"]
|
| 102 |
-
|
| 103 |
def _stream_generate_impl(self, prompt: str, conversation_history: List[Dict]) -> List[str]:
|
| 104 |
-
"""Implementation of streaming generation"""
|
| 105 |
url = f"{self.host}/api/chat"
|
| 106 |
messages = conversation_history.copy()
|
| 107 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
# Add the current prompt if not already in history
|
| 109 |
if not messages or messages[-1].get("content") != prompt:
|
| 110 |
-
|
| 111 |
-
|
| 112 |
payload = {
|
| 113 |
"model": self.model_name,
|
| 114 |
-
"messages":
|
| 115 |
"stream": True
|
| 116 |
}
|
| 117 |
|
|
@@ -135,3 +154,14 @@ class OllamaProvider(LLMProvider):
|
|
| 135 |
except:
|
| 136 |
continue
|
| 137 |
return chunks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
from typing import List, Dict, Optional, Union
|
| 6 |
from core.providers.base import LLMProvider
|
| 7 |
from utils.config import config
|
| 8 |
+
from services.weather import weather_service
|
| 9 |
|
| 10 |
logger = logging.getLogger(__name__)
|
| 11 |
|
| 12 |
class OllamaProvider(LLMProvider):
|
| 13 |
"""Ollama LLM provider implementation"""
|
| 14 |
|
| 15 |
+
def __init__(self, model_name: str, timeout: int = 60, max_retries: int = 3):
|
| 16 |
+
# Increased timeout from 30 to 60
|
| 17 |
super().__init__(model_name, timeout, max_retries)
|
| 18 |
self.host = self._sanitize_host(config.ollama_host or "http://localhost:11434")
|
| 19 |
# Headers to skip ngrok browser warning
|
|
|
|
| 21 |
"ngrok-skip-browser-warning": "true",
|
| 22 |
"User-Agent": "CosmicCat-AI-Assistant"
|
| 23 |
}
|
| 24 |
+
|
| 25 |
def _sanitize_host(self, host: str) -> str:
|
| 26 |
"""Sanitize host URL by removing whitespace and control characters"""
|
| 27 |
if not host:
|
|
|
|
| 34 |
if not host.startswith(('http://', 'https://')):
|
| 35 |
host = 'http://' + host
|
| 36 |
return host
|
| 37 |
+
|
| 38 |
def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
|
| 39 |
"""Generate a response synchronously"""
|
| 40 |
try:
|
|
|
|
| 42 |
except Exception as e:
|
| 43 |
logger.error(f"Ollama generation failed: {e}")
|
| 44 |
return None
|
| 45 |
+
|
| 46 |
def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
|
| 47 |
"""Generate a response with streaming support"""
|
| 48 |
try:
|
|
|
|
| 50 |
except Exception as e:
|
| 51 |
logger.error(f"Ollama stream generation failed: {e}")
|
| 52 |
return None
|
| 53 |
+
|
| 54 |
def validate_model(self) -> bool:
|
| 55 |
"""Validate if the model is available"""
|
| 56 |
try:
|
|
|
|
| 75 |
except Exception as e:
|
| 76 |
logger.error(f"Model validation failed: {e}")
|
| 77 |
return False
|
| 78 |
+
|
| 79 |
def _generate_impl(self, prompt: str, conversation_history: List[Dict]) -> str:
|
| 80 |
+
"""Implementation of synchronous generation with context injection"""
|
| 81 |
url = f"{self.host}/api/chat"
|
| 82 |
messages = conversation_history.copy()
|
| 83 |
|
| 84 |
+
# Inject context message with current time/date/weather
|
| 85 |
+
current_time = datetime.now().strftime("%A, %B %d, %Y at %I:%M %p")
|
| 86 |
+
weather_summary = self._get_weather_summary()
|
| 87 |
+
context_msg = {
|
| 88 |
+
"role": "system",
|
| 89 |
+
"content": f"[Current Context: {current_time} | Weather: {weather_summary}]"
|
| 90 |
+
}
|
| 91 |
+
enhanced_messages = [context_msg] + messages
|
| 92 |
+
|
| 93 |
# Add the current prompt if not already in history
|
| 94 |
if not messages or messages[-1].get("content") != prompt:
|
| 95 |
+
enhanced_messages.append({"role": "user", "content": prompt})
|
| 96 |
+
|
| 97 |
payload = {
|
| 98 |
"model": self.model_name,
|
| 99 |
+
"messages": enhanced_messages,
|
| 100 |
"stream": False
|
| 101 |
}
|
| 102 |
|
|
|
|
| 109 |
response.raise_for_status()
|
| 110 |
result = response.json()
|
| 111 |
return result["message"]["content"]
|
| 112 |
+
|
| 113 |
def _stream_generate_impl(self, prompt: str, conversation_history: List[Dict]) -> List[str]:
|
| 114 |
+
"""Implementation of streaming generation with context injection"""
|
| 115 |
url = f"{self.host}/api/chat"
|
| 116 |
messages = conversation_history.copy()
|
| 117 |
|
| 118 |
+
# Inject context message with current time/date/weather
|
| 119 |
+
current_time = datetime.now().strftime("%A, %B %d, %Y at %I:%M %p")
|
| 120 |
+
weather_summary = self._get_weather_summary()
|
| 121 |
+
context_msg = {
|
| 122 |
+
"role": "system",
|
| 123 |
+
"content": f"[Current Context: {current_time} | Weather: {weather_summary}]"
|
| 124 |
+
}
|
| 125 |
+
enhanced_messages = [context_msg] + messages
|
| 126 |
+
|
| 127 |
# Add the current prompt if not already in history
|
| 128 |
if not messages or messages[-1].get("content") != prompt:
|
| 129 |
+
enhanced_messages.append({"role": "user", "content": prompt})
|
| 130 |
+
|
| 131 |
payload = {
|
| 132 |
"model": self.model_name,
|
| 133 |
+
"messages": enhanced_messages,
|
| 134 |
"stream": True
|
| 135 |
}
|
| 136 |
|
|
|
|
| 154 |
except:
|
| 155 |
continue
|
| 156 |
return chunks
|
| 157 |
+
|
| 158 |
+
def _get_weather_summary(self) -> str:
|
| 159 |
+
"""Get formatted weather summary"""
|
| 160 |
+
try:
|
| 161 |
+
weather = weather_service.get_current_weather("New York")
|
| 162 |
+
if weather:
|
| 163 |
+
return f"{weather.get('temperature', 'N/A')}°C, {weather.get('description', 'Clear skies')}"
|
| 164 |
+
else:
|
| 165 |
+
return "Clear skies"
|
| 166 |
+
except:
|
| 167 |
+
return "Clear skies"
|
services/hf_endpoint_monitor.py
CHANGED
|
@@ -3,12 +3,11 @@ import time
|
|
| 3 |
import logging
|
| 4 |
from typing import Dict, Optional
|
| 5 |
from utils.config import config
|
| 6 |
-
|
| 7 |
logger = logging.getLogger(__name__)
|
| 8 |
|
| 9 |
class HFEndpointMonitor:
|
| 10 |
"""Monitor Hugging Face endpoint status and health"""
|
| 11 |
-
|
| 12 |
def __init__(self):
|
| 13 |
# Clean the endpoint URL
|
| 14 |
raw_url = config.hf_api_url or ""
|
|
@@ -23,38 +22,31 @@ class HFEndpointMonitor:
|
|
| 23 |
self.successful_requests = 0
|
| 24 |
self.failed_requests = 0
|
| 25 |
self.avg_response_time = 0
|
| 26 |
-
|
| 27 |
logger.info(f"Initialized HF Monitor with URL: {self.endpoint_url}")
|
| 28 |
-
|
| 29 |
def _clean_endpoint_url(self, url: str) -> str:
|
| 30 |
"""Clean and validate endpoint URL"""
|
| 31 |
if not url:
|
| 32 |
return ""
|
| 33 |
-
|
| 34 |
# Remove environment variable names if present
|
| 35 |
url = url.replace('hf_api_endpoint_url=', '')
|
| 36 |
url = url.replace('HF_API_ENDPOINT_URL=', '')
|
| 37 |
-
|
| 38 |
# Strip whitespace
|
| 39 |
url = url.strip()
|
| 40 |
-
|
| 41 |
# Ensure it starts with https://
|
| 42 |
if url and not url.startswith(('http://', 'https://')):
|
| 43 |
if 'huggingface.cloud' in url:
|
| 44 |
url = 'https://' + url
|
| 45 |
else:
|
| 46 |
url = 'https://' + url
|
| 47 |
-
|
| 48 |
# Remove trailing slashes but keep /v1 if present
|
| 49 |
if url.endswith('/'):
|
| 50 |
url = url.rstrip('/')
|
| 51 |
-
|
| 52 |
return url
|
| 53 |
-
|
| 54 |
def check_endpoint_status(self) -> Dict:
|
| 55 |
"""Check if HF endpoint is available and initialized with rate limiting"""
|
| 56 |
current_time = time.time()
|
| 57 |
-
|
| 58 |
# Don't check too frequently - minimum 1 minute between checks
|
| 59 |
if current_time - self.last_check < 60:
|
| 60 |
# Return cached status or basic status
|
|
@@ -64,10 +56,8 @@ class HFEndpointMonitor:
|
|
| 64 |
'initialized': getattr(self, '_last_initialized', False),
|
| 65 |
'timestamp': self.last_check
|
| 66 |
}
|
| 67 |
-
|
| 68 |
# Proceed with actual check
|
| 69 |
self.last_check = current_time
|
| 70 |
-
|
| 71 |
try:
|
| 72 |
if not self.endpoint_url or not self.hf_token:
|
| 73 |
status_info = {
|
|
@@ -81,15 +71,12 @@ class HFEndpointMonitor:
|
|
| 81 |
# Properly construct the models endpoint URL
|
| 82 |
models_url = f"{self.endpoint_url.rstrip('/')}/models"
|
| 83 |
logger.info(f"Checking HF endpoint at: {models_url}")
|
| 84 |
-
|
| 85 |
headers = {"Authorization": f"Bearer {self.hf_token}"}
|
| 86 |
-
|
| 87 |
response = requests.get(
|
| 88 |
models_url,
|
| 89 |
headers=headers,
|
| 90 |
timeout=15
|
| 91 |
)
|
| 92 |
-
|
| 93 |
status_info = {
|
| 94 |
'available': response.status_code in [200, 201],
|
| 95 |
'status_code': response.status_code,
|
|
@@ -97,23 +84,34 @@ class HFEndpointMonitor:
|
|
| 97 |
'response_time': response.elapsed.total_seconds(),
|
| 98 |
'timestamp': time.time()
|
| 99 |
}
|
| 100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
if response.status_code not in [200, 201]:
|
| 102 |
status_info['error'] = f"HTTP {response.status_code}: {response.text[:200]}"
|
| 103 |
-
|
| 104 |
logger.info(f"HF Endpoint Status: {status_info}")
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
self._last_initialized = status_info.get('initialized', False)
|
| 110 |
-
|
| 111 |
return status_info
|
| 112 |
-
|
| 113 |
except Exception as e:
|
| 114 |
error_msg = str(e)
|
| 115 |
logger.error(f"HF endpoint check failed: {error_msg}")
|
| 116 |
-
|
| 117 |
status_info = {
|
| 118 |
'available': False,
|
| 119 |
'status_code': None,
|
|
@@ -121,14 +119,12 @@ class HFEndpointMonitor:
|
|
| 121 |
'error': error_msg,
|
| 122 |
'timestamp': time.time()
|
| 123 |
}
|
| 124 |
-
|
| 125 |
# Cache the results
|
| 126 |
self._last_available = False
|
| 127 |
self._last_status_code = None
|
| 128 |
self._last_initialized = False
|
| 129 |
-
|
| 130 |
return status_info
|
| 131 |
-
|
| 132 |
def _is_endpoint_initialized(self, response) -> bool:
|
| 133 |
"""Determine if endpoint is fully initialized"""
|
| 134 |
try:
|
|
@@ -136,40 +132,34 @@ class HFEndpointMonitor:
|
|
| 136 |
return 'data' in data or 'models' in data
|
| 137 |
except:
|
| 138 |
return response.status_code in [200, 201]
|
| 139 |
-
|
| 140 |
def warm_up_endpoint(self) -> bool:
|
| 141 |
"""Send a warm-up request to initialize the endpoint"""
|
| 142 |
try:
|
| 143 |
if not self.endpoint_url or not self.hf_token:
|
| 144 |
logger.warning("Cannot warm up HF endpoint - URL or token not configured")
|
| 145 |
return False
|
| 146 |
-
|
| 147 |
self.warmup_attempts += 1
|
| 148 |
logger.info(f"Warming up HF endpoint (attempt {self.warmup_attempts})...")
|
| 149 |
-
|
| 150 |
headers = {
|
| 151 |
"Authorization": f"Bearer {self.hf_token}",
|
| 152 |
"Content-Type": "application/json"
|
| 153 |
}
|
| 154 |
-
|
| 155 |
# Construct proper chat completions URL
|
| 156 |
chat_url = f"{self.endpoint_url.rstrip('/')}/chat/completions"
|
| 157 |
logger.info(f"Sending warm-up request to: {chat_url}")
|
| 158 |
-
|
| 159 |
payload = {
|
| 160 |
-
"model": "
|
| 161 |
"messages": [{"role": "user", "content": "Hello"}],
|
| 162 |
"max_tokens": 10,
|
| 163 |
"stream": False
|
| 164 |
}
|
| 165 |
-
|
| 166 |
response = requests.post(
|
| 167 |
chat_url,
|
| 168 |
headers=headers,
|
| 169 |
json=payload,
|
| 170 |
timeout=45 # Longer timeout for cold start
|
| 171 |
)
|
| 172 |
-
|
| 173 |
success = response.status_code in [200, 201]
|
| 174 |
if success:
|
| 175 |
self.is_initialized = True
|
|
@@ -179,14 +169,12 @@ class HFEndpointMonitor:
|
|
| 179 |
else:
|
| 180 |
logger.warning(f"⚠️ HF endpoint warm-up response: {response.status_code}")
|
| 181 |
logger.debug(f"Response body: {response.text[:500]}")
|
| 182 |
-
|
| 183 |
return success
|
| 184 |
-
|
| 185 |
except Exception as e:
|
| 186 |
logger.error(f"HF endpoint warm-up failed: {e}")
|
| 187 |
self.failed_requests += 1
|
| 188 |
return False
|
| 189 |
-
|
| 190 |
def get_status_summary(self) -> str:
|
| 191 |
"""Get human-readable status summary"""
|
| 192 |
status = self.check_endpoint_status()
|
|
@@ -197,11 +185,10 @@ class HFEndpointMonitor:
|
|
| 197 |
return "🟡 HF Endpoint: Available but Initializing"
|
| 198 |
else:
|
| 199 |
return "🔴 HF Endpoint: Unavailable"
|
| 200 |
-
|
| 201 |
def handle_scale_to_zero(self) -> bool:
|
| 202 |
"""Handle scale-to-zero behavior with user feedback"""
|
| 203 |
logger.info("HF endpoint appears to be scaled to zero. Attempting to wake it up...")
|
| 204 |
-
|
| 205 |
# Try to warm up the endpoint
|
| 206 |
for attempt in range(self.max_warmup_attempts):
|
| 207 |
logger.info(f"Wake-up attempt {attempt + 1}/{self.max_warmup_attempts}")
|
|
@@ -209,15 +196,13 @@ class HFEndpointMonitor:
|
|
| 209 |
logger.info("✅ HF endpoint successfully woken up!")
|
| 210 |
return True
|
| 211 |
time.sleep(10) # Wait between attempts
|
| 212 |
-
|
| 213 |
logger.error("❌ Failed to wake up HF endpoint after all attempts")
|
| 214 |
return False
|
| 215 |
-
|
| 216 |
def get_detailed_status(self) -> Dict:
|
| 217 |
"""Get detailed HF endpoint status with metrics"""
|
| 218 |
try:
|
| 219 |
headers = {"Authorization": f"Bearer {self.hf_token}"}
|
| 220 |
-
|
| 221 |
# Get model info
|
| 222 |
models_url = f"{self.endpoint_url.rstrip('/')}/models"
|
| 223 |
model_response = requests.get(
|
|
@@ -225,7 +210,6 @@ class HFEndpointMonitor:
|
|
| 225 |
headers=headers,
|
| 226 |
timeout=10
|
| 227 |
)
|
| 228 |
-
|
| 229 |
# Get endpoint info if available
|
| 230 |
endpoint_info = {}
|
| 231 |
try:
|
|
@@ -239,7 +223,6 @@ class HFEndpointMonitor:
|
|
| 239 |
endpoint_info = info_response.json()
|
| 240 |
except:
|
| 241 |
pass
|
| 242 |
-
|
| 243 |
status_info = {
|
| 244 |
'available': model_response.status_code == 200,
|
| 245 |
'status_code': model_response.status_code,
|
|
@@ -249,9 +232,7 @@ class HFEndpointMonitor:
|
|
| 249 |
'warmup_attempts': getattr(self, 'warmup_attempts', 0),
|
| 250 |
'is_warming_up': getattr(self, 'is_warming_up', False)
|
| 251 |
}
|
| 252 |
-
|
| 253 |
return status_info
|
| 254 |
-
|
| 255 |
except Exception as e:
|
| 256 |
return {
|
| 257 |
'available': False,
|
|
@@ -260,7 +241,7 @@ class HFEndpointMonitor:
|
|
| 260 |
'error': str(e),
|
| 261 |
'last_checked': time.time()
|
| 262 |
}
|
| 263 |
-
|
| 264 |
def get_performance_metrics(self) -> Dict:
|
| 265 |
"""Get HF endpoint performance metrics"""
|
| 266 |
return {
|
|
@@ -269,12 +250,11 @@ class HFEndpointMonitor:
|
|
| 269 |
'failed_requests': getattr(self, 'failed_requests', 0),
|
| 270 |
'average_response_time': getattr(self, 'avg_response_time', 0)
|
| 271 |
}
|
| 272 |
-
|
| 273 |
# Add enhanced status tracking methods
|
| 274 |
def get_enhanced_status(self) -> Dict:
|
| 275 |
"""Get enhanced HF endpoint status with engagement tracking"""
|
| 276 |
basic_status = self.check_endpoint_status()
|
| 277 |
-
|
| 278 |
return {
|
| 279 |
**basic_status,
|
| 280 |
"engagement_level": self._determine_engagement_level(),
|
|
@@ -282,7 +262,7 @@ class HFEndpointMonitor:
|
|
| 282 |
"total_engagements": getattr(self, '_total_engagements', 0),
|
| 283 |
"current_research_topic": getattr(self, '_current_research_topic', None)
|
| 284 |
}
|
| 285 |
-
|
| 286 |
def _determine_engagement_level(self) -> str:
|
| 287 |
"""Determine current engagement level"""
|
| 288 |
if not self.is_initialized:
|
|
@@ -293,7 +273,7 @@ class HFEndpointMonitor:
|
|
| 293 |
return "research_pending"
|
| 294 |
else:
|
| 295 |
return "ready"
|
| 296 |
-
|
| 297 |
def start_hf_analysis(self, topic: str = None):
|
| 298 |
"""Start HF analysis with topic tracking"""
|
| 299 |
self._currently_analyzing = True
|
|
@@ -301,7 +281,7 @@ class HFEndpointMonitor:
|
|
| 301 |
self._total_engagements = getattr(self, '_total_engagements', 0) + 1
|
| 302 |
if topic:
|
| 303 |
self._current_research_topic = topic
|
| 304 |
-
|
| 305 |
def finish_hf_analysis(self):
|
| 306 |
"""Finish HF analysis"""
|
| 307 |
self._currently_analyzing = False
|
|
|
|
| 3 |
import logging
|
| 4 |
from typing import Dict, Optional
|
| 5 |
from utils.config import config
|
|
|
|
| 6 |
logger = logging.getLogger(__name__)
|
| 7 |
|
| 8 |
class HFEndpointMonitor:
|
| 9 |
"""Monitor Hugging Face endpoint status and health"""
|
| 10 |
+
|
| 11 |
def __init__(self):
|
| 12 |
# Clean the endpoint URL
|
| 13 |
raw_url = config.hf_api_url or ""
|
|
|
|
| 22 |
self.successful_requests = 0
|
| 23 |
self.failed_requests = 0
|
| 24 |
self.avg_response_time = 0
|
|
|
|
| 25 |
logger.info(f"Initialized HF Monitor with URL: {self.endpoint_url}")
|
| 26 |
+
|
| 27 |
def _clean_endpoint_url(self, url: str) -> str:
|
| 28 |
"""Clean and validate endpoint URL"""
|
| 29 |
if not url:
|
| 30 |
return ""
|
|
|
|
| 31 |
# Remove environment variable names if present
|
| 32 |
url = url.replace('hf_api_endpoint_url=', '')
|
| 33 |
url = url.replace('HF_API_ENDPOINT_URL=', '')
|
|
|
|
| 34 |
# Strip whitespace
|
| 35 |
url = url.strip()
|
|
|
|
| 36 |
# Ensure it starts with https://
|
| 37 |
if url and not url.startswith(('http://', 'https://')):
|
| 38 |
if 'huggingface.cloud' in url:
|
| 39 |
url = 'https://' + url
|
| 40 |
else:
|
| 41 |
url = 'https://' + url
|
|
|
|
| 42 |
# Remove trailing slashes but keep /v1 if present
|
| 43 |
if url.endswith('/'):
|
| 44 |
url = url.rstrip('/')
|
|
|
|
| 45 |
return url
|
| 46 |
+
|
| 47 |
def check_endpoint_status(self) -> Dict:
|
| 48 |
"""Check if HF endpoint is available and initialized with rate limiting"""
|
| 49 |
current_time = time.time()
|
|
|
|
| 50 |
# Don't check too frequently - minimum 1 minute between checks
|
| 51 |
if current_time - self.last_check < 60:
|
| 52 |
# Return cached status or basic status
|
|
|
|
| 56 |
'initialized': getattr(self, '_last_initialized', False),
|
| 57 |
'timestamp': self.last_check
|
| 58 |
}
|
|
|
|
| 59 |
# Proceed with actual check
|
| 60 |
self.last_check = current_time
|
|
|
|
| 61 |
try:
|
| 62 |
if not self.endpoint_url or not self.hf_token:
|
| 63 |
status_info = {
|
|
|
|
| 71 |
# Properly construct the models endpoint URL
|
| 72 |
models_url = f"{self.endpoint_url.rstrip('/')}/models"
|
| 73 |
logger.info(f"Checking HF endpoint at: {models_url}")
|
|
|
|
| 74 |
headers = {"Authorization": f"Bearer {self.hf_token}"}
|
|
|
|
| 75 |
response = requests.get(
|
| 76 |
models_url,
|
| 77 |
headers=headers,
|
| 78 |
timeout=15
|
| 79 |
)
|
|
|
|
| 80 |
status_info = {
|
| 81 |
'available': response.status_code in [200, 201],
|
| 82 |
'status_code': response.status_code,
|
|
|
|
| 84 |
'response_time': response.elapsed.total_seconds(),
|
| 85 |
'timestamp': time.time()
|
| 86 |
}
|
| 87 |
+
|
| 88 |
+
# Enhanced status info with model and region if available
|
| 89 |
+
if response.status_code in [200, 201]:
|
| 90 |
+
try:
|
| 91 |
+
data = response.json()
|
| 92 |
+
if 'data' in data and len(data['data']) > 0:
|
| 93 |
+
status_info['model'] = data['data'][0].get('id', 'Unknown')
|
| 94 |
+
# Try to extract region from URL if possible
|
| 95 |
+
if 'us-east-1' in self.endpoint_url:
|
| 96 |
+
status_info['region'] = 'us-east-1'
|
| 97 |
+
elif 'us-west' in self.endpoint_url:
|
| 98 |
+
status_info['region'] = 'us-west'
|
| 99 |
+
except:
|
| 100 |
+
pass
|
| 101 |
+
|
| 102 |
+
status_info['warmup_count'] = getattr(self, 'warmup_count', 0)
|
| 103 |
+
|
| 104 |
if response.status_code not in [200, 201]:
|
| 105 |
status_info['error'] = f"HTTP {response.status_code}: {response.text[:200]}"
|
|
|
|
| 106 |
logger.info(f"HF Endpoint Status: {status_info}")
|
| 107 |
+
# Cache the results
|
| 108 |
+
self._last_available = status_info['available']
|
| 109 |
+
self._last_status_code = status_info['status_code']
|
| 110 |
+
self._last_initialized = status_info.get('initialized', False)
|
|
|
|
|
|
|
| 111 |
return status_info
|
|
|
|
| 112 |
except Exception as e:
|
| 113 |
error_msg = str(e)
|
| 114 |
logger.error(f"HF endpoint check failed: {error_msg}")
|
|
|
|
| 115 |
status_info = {
|
| 116 |
'available': False,
|
| 117 |
'status_code': None,
|
|
|
|
| 119 |
'error': error_msg,
|
| 120 |
'timestamp': time.time()
|
| 121 |
}
|
|
|
|
| 122 |
# Cache the results
|
| 123 |
self._last_available = False
|
| 124 |
self._last_status_code = None
|
| 125 |
self._last_initialized = False
|
|
|
|
| 126 |
return status_info
|
| 127 |
+
|
| 128 |
def _is_endpoint_initialized(self, response) -> bool:
|
| 129 |
"""Determine if endpoint is fully initialized"""
|
| 130 |
try:
|
|
|
|
| 132 |
return 'data' in data or 'models' in data
|
| 133 |
except:
|
| 134 |
return response.status_code in [200, 201]
|
| 135 |
+
|
| 136 |
def warm_up_endpoint(self) -> bool:
|
| 137 |
"""Send a warm-up request to initialize the endpoint"""
|
| 138 |
try:
|
| 139 |
if not self.endpoint_url or not self.hf_token:
|
| 140 |
logger.warning("Cannot warm up HF endpoint - URL or token not configured")
|
| 141 |
return False
|
|
|
|
| 142 |
self.warmup_attempts += 1
|
| 143 |
logger.info(f"Warming up HF endpoint (attempt {self.warmup_attempts})...")
|
|
|
|
| 144 |
headers = {
|
| 145 |
"Authorization": f"Bearer {self.hf_token}",
|
| 146 |
"Content-Type": "application/json"
|
| 147 |
}
|
|
|
|
| 148 |
# Construct proper chat completions URL
|
| 149 |
chat_url = f"{self.endpoint_url.rstrip('/')}/chat/completions"
|
| 150 |
logger.info(f"Sending warm-up request to: {chat_url}")
|
|
|
|
| 151 |
payload = {
|
| 152 |
+
"model": "meta-llama/Llama-2-7b-chat-hf",
|
| 153 |
"messages": [{"role": "user", "content": "Hello"}],
|
| 154 |
"max_tokens": 10,
|
| 155 |
"stream": False
|
| 156 |
}
|
|
|
|
| 157 |
response = requests.post(
|
| 158 |
chat_url,
|
| 159 |
headers=headers,
|
| 160 |
json=payload,
|
| 161 |
timeout=45 # Longer timeout for cold start
|
| 162 |
)
|
|
|
|
| 163 |
success = response.status_code in [200, 201]
|
| 164 |
if success:
|
| 165 |
self.is_initialized = True
|
|
|
|
| 169 |
else:
|
| 170 |
logger.warning(f"⚠️ HF endpoint warm-up response: {response.status_code}")
|
| 171 |
logger.debug(f"Response body: {response.text[:500]}")
|
|
|
|
| 172 |
return success
|
|
|
|
| 173 |
except Exception as e:
|
| 174 |
logger.error(f"HF endpoint warm-up failed: {e}")
|
| 175 |
self.failed_requests += 1
|
| 176 |
return False
|
| 177 |
+
|
| 178 |
def get_status_summary(self) -> str:
|
| 179 |
"""Get human-readable status summary"""
|
| 180 |
status = self.check_endpoint_status()
|
|
|
|
| 185 |
return "🟡 HF Endpoint: Available but Initializing"
|
| 186 |
else:
|
| 187 |
return "🔴 HF Endpoint: Unavailable"
|
| 188 |
+
|
| 189 |
def handle_scale_to_zero(self) -> bool:
|
| 190 |
"""Handle scale-to-zero behavior with user feedback"""
|
| 191 |
logger.info("HF endpoint appears to be scaled to zero. Attempting to wake it up...")
|
|
|
|
| 192 |
# Try to warm up the endpoint
|
| 193 |
for attempt in range(self.max_warmup_attempts):
|
| 194 |
logger.info(f"Wake-up attempt {attempt + 1}/{self.max_warmup_attempts}")
|
|
|
|
| 196 |
logger.info("✅ HF endpoint successfully woken up!")
|
| 197 |
return True
|
| 198 |
time.sleep(10) # Wait between attempts
|
|
|
|
| 199 |
logger.error("❌ Failed to wake up HF endpoint after all attempts")
|
| 200 |
return False
|
| 201 |
+
|
| 202 |
def get_detailed_status(self) -> Dict:
|
| 203 |
"""Get detailed HF endpoint status with metrics"""
|
| 204 |
try:
|
| 205 |
headers = {"Authorization": f"Bearer {self.hf_token}"}
|
|
|
|
| 206 |
# Get model info
|
| 207 |
models_url = f"{self.endpoint_url.rstrip('/')}/models"
|
| 208 |
model_response = requests.get(
|
|
|
|
| 210 |
headers=headers,
|
| 211 |
timeout=10
|
| 212 |
)
|
|
|
|
| 213 |
# Get endpoint info if available
|
| 214 |
endpoint_info = {}
|
| 215 |
try:
|
|
|
|
| 223 |
endpoint_info = info_response.json()
|
| 224 |
except:
|
| 225 |
pass
|
|
|
|
| 226 |
status_info = {
|
| 227 |
'available': model_response.status_code == 200,
|
| 228 |
'status_code': model_response.status_code,
|
|
|
|
| 232 |
'warmup_attempts': getattr(self, 'warmup_attempts', 0),
|
| 233 |
'is_warming_up': getattr(self, 'is_warming_up', False)
|
| 234 |
}
|
|
|
|
| 235 |
return status_info
|
|
|
|
| 236 |
except Exception as e:
|
| 237 |
return {
|
| 238 |
'available': False,
|
|
|
|
| 241 |
'error': str(e),
|
| 242 |
'last_checked': time.time()
|
| 243 |
}
|
| 244 |
+
|
| 245 |
def get_performance_metrics(self) -> Dict:
|
| 246 |
"""Get HF endpoint performance metrics"""
|
| 247 |
return {
|
|
|
|
| 250 |
'failed_requests': getattr(self, 'failed_requests', 0),
|
| 251 |
'average_response_time': getattr(self, 'avg_response_time', 0)
|
| 252 |
}
|
| 253 |
+
|
| 254 |
# Add enhanced status tracking methods
|
| 255 |
def get_enhanced_status(self) -> Dict:
|
| 256 |
"""Get enhanced HF endpoint status with engagement tracking"""
|
| 257 |
basic_status = self.check_endpoint_status()
|
|
|
|
| 258 |
return {
|
| 259 |
**basic_status,
|
| 260 |
"engagement_level": self._determine_engagement_level(),
|
|
|
|
| 262 |
"total_engagements": getattr(self, '_total_engagements', 0),
|
| 263 |
"current_research_topic": getattr(self, '_current_research_topic', None)
|
| 264 |
}
|
| 265 |
+
|
| 266 |
def _determine_engagement_level(self) -> str:
|
| 267 |
"""Determine current engagement level"""
|
| 268 |
if not self.is_initialized:
|
|
|
|
| 273 |
return "research_pending"
|
| 274 |
else:
|
| 275 |
return "ready"
|
| 276 |
+
|
| 277 |
def start_hf_analysis(self, topic: str = None):
|
| 278 |
"""Start HF analysis with topic tracking"""
|
| 279 |
self._currently_analyzing = True
|
|
|
|
| 281 |
self._total_engagements = getattr(self, '_total_engagements', 0) + 1
|
| 282 |
if topic:
|
| 283 |
self._current_research_topic = topic
|
| 284 |
+
|
| 285 |
def finish_hf_analysis(self):
|
| 286 |
"""Finish HF analysis"""
|
| 287 |
self._currently_analyzing = False
|
services/weather.py
CHANGED
|
@@ -11,15 +11,7 @@ class WeatherService:
|
|
| 11 |
self.base_url = "http://api.openweathermap.org/data/2.5"
|
| 12 |
|
| 13 |
def get_current_weather(self, city: str) -> Optional[Dict[str, Any]]:
|
| 14 |
-
"""
|
| 15 |
-
Get current weather for a city
|
| 16 |
-
|
| 17 |
-
Args:
|
| 18 |
-
city: Name of the city
|
| 19 |
-
|
| 20 |
-
Returns:
|
| 21 |
-
Dictionary with weather information or None if failed
|
| 22 |
-
"""
|
| 23 |
if not self.api_key:
|
| 24 |
print("OpenWeather API key not configured")
|
| 25 |
return None
|
|
@@ -30,13 +22,11 @@ class WeatherService:
|
|
| 30 |
'appid': self.api_key,
|
| 31 |
'units': 'metric' # Celsius
|
| 32 |
}
|
| 33 |
-
|
| 34 |
response = requests.get(
|
| 35 |
f"{self.base_url}/weather",
|
| 36 |
params=params,
|
| 37 |
timeout=10
|
| 38 |
)
|
| 39 |
-
|
| 40 |
if response.status_code == 200:
|
| 41 |
data = response.json()
|
| 42 |
return {
|
|
@@ -51,22 +41,12 @@ class WeatherService:
|
|
| 51 |
else:
|
| 52 |
print(f"Weather API error: {response.status_code} - {response.text}")
|
| 53 |
return None
|
| 54 |
-
|
| 55 |
except Exception as e:
|
| 56 |
print(f"Error fetching weather data: {e}")
|
| 57 |
return None
|
| 58 |
-
|
| 59 |
-
def get_forecast(self, city: str, days: int = 5) -> Optional[Dict[str, Any]]:
|
| 60 |
-
"""
|
| 61 |
-
Get weather forecast for a city
|
| 62 |
-
|
| 63 |
-
Args:
|
| 64 |
-
city: Name of the city
|
| 65 |
-
days: Number of days to forecast (default: 5)
|
| 66 |
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
"""
|
| 70 |
if not self.api_key:
|
| 71 |
print("OpenWeather API key not configured")
|
| 72 |
return None
|
|
@@ -78,17 +58,14 @@ class WeatherService:
|
|
| 78 |
'units': 'metric',
|
| 79 |
'cnt': days
|
| 80 |
}
|
| 81 |
-
|
| 82 |
response = requests.get(
|
| 83 |
f"{self.base_url}/forecast",
|
| 84 |
params=params,
|
| 85 |
timeout=10
|
| 86 |
)
|
| 87 |
-
|
| 88 |
if response.status_code == 200:
|
| 89 |
data = response.json()
|
| 90 |
forecasts = []
|
| 91 |
-
|
| 92 |
for item in data['list']:
|
| 93 |
forecasts.append({
|
| 94 |
'datetime': item['dt_txt'],
|
|
@@ -96,7 +73,6 @@ class WeatherService:
|
|
| 96 |
'description': item['weather'][0]['description'],
|
| 97 |
'icon': item['weather'][0]['icon']
|
| 98 |
})
|
| 99 |
-
|
| 100 |
return {
|
| 101 |
'city': data['city']['name'],
|
| 102 |
'country': data['city']['country'],
|
|
@@ -105,10 +81,20 @@ class WeatherService:
|
|
| 105 |
else:
|
| 106 |
print(f"Forecast API error: {response.status_code} - {response.text}")
|
| 107 |
return None
|
| 108 |
-
|
| 109 |
except Exception as e:
|
| 110 |
print(f"Error fetching forecast data: {e}")
|
| 111 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
# Global weather service instance
|
| 114 |
weather_service = WeatherService()
|
|
|
|
| 11 |
self.base_url = "http://api.openweathermap.org/data/2.5"
|
| 12 |
|
| 13 |
def get_current_weather(self, city: str) -> Optional[Dict[str, Any]]:
|
| 14 |
+
"""Get current weather for a city"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
if not self.api_key:
|
| 16 |
print("OpenWeather API key not configured")
|
| 17 |
return None
|
|
|
|
| 22 |
'appid': self.api_key,
|
| 23 |
'units': 'metric' # Celsius
|
| 24 |
}
|
|
|
|
| 25 |
response = requests.get(
|
| 26 |
f"{self.base_url}/weather",
|
| 27 |
params=params,
|
| 28 |
timeout=10
|
| 29 |
)
|
|
|
|
| 30 |
if response.status_code == 200:
|
| 31 |
data = response.json()
|
| 32 |
return {
|
|
|
|
| 41 |
else:
|
| 42 |
print(f"Weather API error: {response.status_code} - {response.text}")
|
| 43 |
return None
|
|
|
|
| 44 |
except Exception as e:
|
| 45 |
print(f"Error fetching weather data: {e}")
|
| 46 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
|
| 48 |
+
def get_forecast(self, city: str, days: int = 5) -> Optional[Dict[str, Any]]:
|
| 49 |
+
"""Get weather forecast for a city"""
|
|
|
|
| 50 |
if not self.api_key:
|
| 51 |
print("OpenWeather API key not configured")
|
| 52 |
return None
|
|
|
|
| 58 |
'units': 'metric',
|
| 59 |
'cnt': days
|
| 60 |
}
|
|
|
|
| 61 |
response = requests.get(
|
| 62 |
f"{self.base_url}/forecast",
|
| 63 |
params=params,
|
| 64 |
timeout=10
|
| 65 |
)
|
|
|
|
| 66 |
if response.status_code == 200:
|
| 67 |
data = response.json()
|
| 68 |
forecasts = []
|
|
|
|
| 69 |
for item in data['list']:
|
| 70 |
forecasts.append({
|
| 71 |
'datetime': item['dt_txt'],
|
|
|
|
| 73 |
'description': item['weather'][0]['description'],
|
| 74 |
'icon': item['weather'][0]['icon']
|
| 75 |
})
|
|
|
|
| 76 |
return {
|
| 77 |
'city': data['city']['name'],
|
| 78 |
'country': data['city']['country'],
|
|
|
|
| 81 |
else:
|
| 82 |
print(f"Forecast API error: {response.status_code} - {response.text}")
|
| 83 |
return None
|
|
|
|
| 84 |
except Exception as e:
|
| 85 |
print(f"Error fetching forecast data: {e}")
|
| 86 |
return None
|
| 87 |
+
|
| 88 |
+
def get_weather_summary(self, city="New York") -> str:
|
| 89 |
+
"""Get formatted weather summary"""
|
| 90 |
+
try:
|
| 91 |
+
weather = self.get_current_weather(city)
|
| 92 |
+
if weather:
|
| 93 |
+
return f"{weather.get('temperature', 'N/A')}°C, {weather.get('description', 'Clear skies')}"
|
| 94 |
+
else:
|
| 95 |
+
return "Clear skies"
|
| 96 |
+
except:
|
| 97 |
+
return "Clear skies"
|
| 98 |
|
| 99 |
# Global weather service instance
|
| 100 |
weather_service = WeatherService()
|