Update services/utils/http_utils.py

services/utils/http_utils.py

@@ -84,16 +84,6 @@ async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) ->
         proxy_url = get_proxy_url()
         logger.info(f"Using proxy {proxy_url} (attempt {attempt+1})")
         
-        # Special adjustments for "hosteller" searches
-        if "hosteller" in url.lower():
-            enhanced_headers["Referer"] = "https://www.google.com/search?q=hosteller+hotels+india"
-            # Remove any location qualifiers for better results
-            if "old+manali" in url.lower():
-                url = url.replace("old+manali", "manali")
-            elif "narkanda" in url.lower() and "hosteller" in url.lower():
-                # Try a more general search for Narkanda hostels
-                url = url.replace("the+hosteller+narkanda", "hostels+in+narkanda")
-        
         try:
             # Use a longer timeout for proxies
             async with session.get(
@@ -109,16 +99,20 @@ async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) ->
                     
                     # Check if we got actual content, not a bot detection page
                     if len(content) > 5000 and ("<html" in content or "<!DOCTYPE" in content):
-                        # If this is a search page, additional validation
+                        # Check for property content on search pages
                         if "searchresults" in url or "search" in url:
-                            if "property-card" in content or "sr_property_block" in content:
+                            if "property-card" in content or "sr_property_block" in content or "sr_item" in content:
                                 logger.info(f"Successfully retrieved search results ({len(content)} bytes)")
                                 return content
                             else:
                                 logger.warning("No property cards found in search results")
+                                mark_proxy_failure(proxy_url)
                         else:
                             logger.info(f"Successfully retrieved content ({len(content)} bytes)")
                             return content
+                    else:
+                        logger.warning(f"Response too short or not HTML: {len(content)} bytes")
+                        mark_proxy_failure(proxy_url)
                 else:
                     logger.warning(f"Response status {response.status} from proxy {proxy_url}")
                     mark_proxy_failure(proxy_url)