Consolidate Telegram webhook fixes and debugging improvements

ai_service.py

@@ -10,17 +10,11 @@ MODEL_NAME = HF_MODEL
 def clean_ai_response(text: str):
     if not text: return ""
     text = re.sub(r'<think>.*?</think>', '', text, flags=re.DOTALL)
-    # Remove HTML tags
     text = re.sub(r'<br\s*/?>', '\n', text)
     text = re.sub(r'<[^>]+>', '', text)
-    # Remove markdown tables (lines that start and end with |)
     text = re.sub(r'^\|.*\|\s*$', '', text, flags=re.MULTILINE)
     text = re.sub(r'^[\s|:-]+$', '', text, flags=re.MULTILINE)
-    # Remove markdown bold/italic
-    text = re.sub(r'[*_]{1,3}(.*?)[*_]{1,3}', r'\1', text)
-    # Remove markdown headers
     text = re.sub(r'^#{1,6}\s*', '', text, flags=re.MULTILINE)
-    # Clean up extra blank lines
     text = re.sub(r'\n{3,}', '\n\n', text)
     return text.strip()
 
@@ -105,7 +99,9 @@ async def get_ai_response(user_query: str, telegram_id: int):
         completion = await loop.run_in_executor(None, lambda: call_hf(messages))
         response_message = completion.choices[0].message
 
-    final_response = clean_ai_response(response_message.content)
+    final_response = clean_ai_response(response_message.content if response_message.content else "")
+    print(f"--- AI Raw Response: {repr(response_message.content)} ---")
+    print(f"--- AI Final Response: {repr(final_response)} ---")
 
     if db_manager:
         db_manager.save_message(telegram_id, user_query, "user")

config.py

@@ -12,10 +12,10 @@ HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HF_API_KEY")
 TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN")
 SUPABASE_URL = os.environ.get("SUPABASE_URL")
 SUPABASE_KEY = os.environ.get("SUPABASE_KEY")
-TELEGRAM_DOMAIN = os.environ.get("TELEGRAM_DOMAIN")
+TELEGRAM_DOMAIN = os.environ.get("TELEGRAM_DOMAIN", "https://api.telegram.org").rstrip("/")
 
 # Only create TELEGRAM_URL if token exists
-TELEGRAM_URL = f"{TELEGRAM_DOMAIN}/bot{TELEGRAM_TOKEN}/sendMessage" if TELEGRAM_TOKEN else None
+TELEGRAM_URL = f"{TELEGRAM_DOMAIN}/bot{TELEGRAM_TOKEN}/sendMessage" if TELEGRAM_TOKEN and TELEGRAM_DOMAIN else None
 
 EMBED_MODEL = os.environ.get("EMBED_MODEL", "multilingual-e5-large")
 HF_MODEL = os.environ.get(

telegram_handlers.py

@@ -1,11 +1,28 @@
-from fastapi import Request
 from pydantic import BaseModel
 import httpx
+import json
 from config import TELEGRAM_URL
 from ai_service import get_ai_response
 from database import db_manager
 
 TELEGRAM_IP = "149.154.167.220"
+MAX_TELEGRAM_MESSAGE_LENGTH = 4096
+
+
+def _sanitize_telegram_text(text: str) -> str:
+    if text is None:
+        return ""
+
+    normalized = str(text).replace("\r\n", "\n").replace("\r", "\n")
+
+    # Remove control/surrogate chars that Telegram can reject.
+    cleaned = "".join(
+        ch
+        for ch in normalized
+        if (ch in ("\n", "\t") or ord(ch) >= 32) and not (0xD800 <= ord(ch) <= 0xDFFF)
+    )
+    return cleaned.strip()
+
 
 class ChatInfo(BaseModel):
     id: int
@@ -35,42 +52,59 @@ async def telegram_webhook(data: WebhookData):
 
         if db_manager:
             db_manager.create_or_update_user(telegram_id, username, first_name, data.message.chat.last_name)
-            db_manager.save_message(telegram_id, user_text, "user")
+
         
         ai_answer = await get_ai_response(user_text, telegram_id)
         
-        if db_manager:
-            db_manager.save_message(telegram_id, ai_answer, "assistant")
+
         
         if TELEGRAM_URL:
             try:
                 from config import TELEGRAM_TOKEN 
                 
-                async with httpx.AsyncClient(timeout=40.0, verify=False, follow_redirects=True) as client:
+                async with httpx.AsyncClient(timeout=40.0, verify=False) as client:
+                    prepared_text = _sanitize_telegram_text(
+                        ai_answer or "Sorry, I couldn't generate a response. Please try again."
+                    )
+                    if not prepared_text:
+                        prepared_text = "Sorry, I couldn't generate a response. Please try again."
+
+                    final_text = prepared_text[:MAX_TELEGRAM_MESSAGE_LENGTH]
                     payload = {
                         "chat_id": telegram_id,
-                        "text": ai_answer,
-                        "parse_mode": "Markdown"
+                        "text": final_text if final_text.strip() else ".",
                     }
-                    
-                    try:
-                        response = await client.post(TELEGRAM_URL, json=payload)
-                    except Exception as dns_err:
-                        print(f"--- DNS Failed. Forcing Direct IP Routing to 149.154.167.220 ---")
-                        
-                        forced_ip_url = f"https://149.154.167.220/bot{TELEGRAM_TOKEN}/sendMessage"
-                        
+
+                    print(f"--- Sending Telegram message (chars={len(payload['text'])}) ---")
+
+                    try:                        
+                        response = await client.post(TELEGRAM_URL, data=payload)
+                    except Exception:
+                        print(f"--- DNS Failed. Forcing Direct IP Routing to {TELEGRAM_IP} ---")
+
+                        forced_ip_url = f"https://{TELEGRAM_IP}/bot{TELEGRAM_TOKEN}/sendMessage"
+                        json_body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
                         headers = {
                             "Host": "api.telegram.org",
-                            "Content-Type": "application/json"
+                            "Content-Type": "application/json; charset=utf-8",
+                            "Content-Length": str(len(json_body)),
                         }
-                        
-                        response = await client.post(forced_ip_url, json=payload, headers=headers)
-                    
+
+                        response = await client.post(
+                            forced_ip_url,
+                            content=json_body,
+                            headers=headers,
+                        )
+
                     if response.status_code == 200:
-                        print(f"--- Success: Message delivered via Direct IP Pipeline ---")
+                        print("--- Success: Telegram message delivered ---")
                     else:
-                        print(f"--- Telegram Rejected Request: {response.status_code} - {response.text} ---")
+                        print("--- Telegram payload rejected ---")
+                        print(f"--- Payload: {payload} ---")
+                        print(
+                            f"--- Telegram Rejected Request: "
+                            f"{response.status_code} - {response.text} ---"
+                        )
                         
             except Exception as send_error:
                 print(f"--- Emergency: Network Blockage Detected: {str(send_error)} ---")
@@ -78,4 +112,4 @@ async def telegram_webhook(data: WebhookData):
         return {"status": "ok"}
     except Exception as e:
         print(f"Error in webhook: {str(e)}")
-        return {"status": "error", "message": str(e)}
+        return {"status": "error", "message": str(e)}