Update app.py

app.py

@@ -17,22 +17,17 @@ class CredentialRevolver:
 
     def _parse_proxies(self, proxy_string: str):
         proxies = []
-        if not proxy_string:
-            return proxies
+        if not proxy_string: return proxies
         for line in proxy_string.strip().splitlines():
             try:
                 parsed = urllib.parse.urlparse(f"//{line.strip()}")
-                if not parsed.hostname or not parsed.port:
-                    continue
+                if not parsed.hostname or not parsed.port: continue
                 server = f"http://{parsed.hostname}:{parsed.port}"
                 proxy_dict = {"server": server}
-                if parsed.username:
-                    proxy_dict["username"] = urllib.parse.unquote(parsed.username)
-                if parsed.password:
-                    proxy_dict["password"] = urllib.parse.unquote(parsed.password)
+                if parsed.username: proxy_dict["username"] = urllib.parse.unquote(parsed.username)
+                if parsed.password: proxy_dict["password"] = urllib.parse.unquote(parsed.password)
                 proxies.append(proxy_dict)
-            except Exception:
-                pass
+            except Exception: pass
         return proxies
 
     def get_next(self) -> Optional[Dict]:
@@ -45,21 +40,21 @@ PLAYWRIGHT_STATE: Dict = {}
 REVOLVER = CredentialRevolver(os.getenv("PROXY_LIST", ""))
 
 SEARCH_ENGINES = {
-    "Google": "https://www.google.com/search?q={query}&hl=en",
-    "DuckDuckGo": "https://duckduckgo.com/html/?q={query}",
-    "Bing": "https://www.bing.com/search?q={query}",
-    "Brave": "https://search.brave.com/search?q={query}",
-    "Ecosia": "https://www.ecosia.org/search?q={query}",
-    "Yahoo": "https://search.yahoo.com/search?p={query}",
-    "Startpage": "https://www.startpage.com/sp/search?q={query}",
-    "Qwant": "https://www.qwant.com/?q={query}",
-    "Swisscows": "https://swisscows.com/web?query={query}",
-    "You.com": "https://you.com/search?q={query}",
-    "SearXNG": "https://searx.be/search?q={query}",
-    "MetaGer": "https://metager.org/meta/meta.ger-en?eingabe={query}",
-    "Yandex": "https://yandex.com/search/?text={query}",
-    "Baidu": "https://www.baidu.com/s?wd={query}",
-    "Perplexity": "https://www.perplexity.ai/search?q={query}"
+    "Google": ("https://www.google.com/search?q={query}&hl=en", '#rso, #search, #botstuff'),
+    "DuckDuckGo": ("https://duckduckgo.com/html/?q={query}", '#links'),
+    "Bing": ("https://www.bing.com/search?q={query}", '#b_results'),
+    "Brave": ("https://search.brave.com/search?q={query}", '#results'),
+    "Ecosia": ("https://www.ecosia.org/search?q={query}", 'main[role="main"]'),
+    "Yahoo": ("https://search.yahoo.com/search?p={query}", '#web'),
+    "Startpage": ("https://www.startpage.com/sp/search?q={query}", '#main'),
+    "Qwant": ("https://www.qwant.com/?q={query}", '[data-testid="web-results"]'),
+    "Swisscows": ("https://swisscows.com/web?query={query}", '.web-results'),
+    "You.com": ("https://you.com/search?q={query}", '#search-results'),
+    "SearXNG": ("https://searx.be/search?q={query}", '#results'),
+    "MetaGer": ("https://metager.org/meta/meta.ger-en?eingabe={query}", '#results'),
+    "Yandex": ("https://yandex.com/search/?text={query}", '#search-result'),
+    "Baidu": ("https://www.baidu.com/s?wd={query}", '#content_left'),
+    "Perplexity": ("https://www.perplexity.ai/search?q={query}", 'div[class*="prose"]'),
 }
 
 class HTML_TO_MARKDOWN_CONVERTER:
@@ -68,13 +63,7 @@ class HTML_TO_MARKDOWN_CONVERTER:
         self.base_url = base_url
 
     def _cleanup_html(self):
-        selectors_to_remove = [
-            'nav', 'footer', 'header', 'aside', 'form', 'script', 'style', 'svg', 'button', 'input', 'textarea',
-            '[role="navigation"]', '[role="search"]', '[id*="comment"]', '[class*="comment-"]',
-            '[id*="sidebar"]', '[class*="sidebar"]', '[id*="related"]', '[class*="related"]',
-            '[id*="share"]', '[class*="share"]', '[id*="social"]', '[class*="social"]',
-            '[id*="cookie"]', '[class*="cookie"]'
-        ]
+        selectors_to_remove = ['nav', 'footer', 'header', 'aside', 'form', 'script', 'style', 'svg', 'button', 'input', 'textarea', '[role="navigation"]', '[role="search"]', '[id*="comment"]', '[class*="comment-"]', '[id*="sidebar"]', '[class*="sidebar"]', '[id*="related"]', '[class*="related"]', '[id*="share"]', '[class*="share"]', '[id*="social"]', '[class*="social"]', '[id*="cookie"]', '[class*="cookie"]', '[aria-hidden="true"]']
         for selector in selectors_to_remove:
             for element in self.soup.select(selector):
                 element.decompose()
@@ -82,8 +71,7 @@ class HTML_TO_MARKDOWN_CONVERTER:
     def convert(self):
         self._cleanup_html()
         content_node = self.soup.find('main') or self.soup.find('article') or self.soup.find('body')
-        if not content_node:
-            return ""
+        if not content_node: return ""
         md = self._process_node(content_node)
         return re.sub(r'\n{3,}', '\n\n', md).strip()
 
@@ -120,7 +108,7 @@ class HTML_TO_MARKDOWN_CONVERTER:
             return f"\n\n![{alt}]({full_src})\n\n"
         return inner_md
 
-async def perform_web_browse(query: str, browser_name: str, search_engine: str):
+async def perform_web_browse(query: str, browser_name: str, search_engine_name: str):
     browser_key = browser_name.lower()
     if "playwright" not in PLAYWRIGHT_STATE:
         PLAYWRIGHT_STATE["playwright"] = await async_playwright().start()
@@ -138,55 +126,52 @@ async def perform_web_browse(query: str, browser_name: str, search_engine: str):
     
     browser_instance = PLAYWRIGHT_STATE[browser_key]
 
-    if urllib.parse.urlparse(query).scheme in ['http', 'https'] and '.' in urllib.parse.urlparse(query).netloc:
+    is_direct_url = urllib.parse.urlparse(query).scheme in ['http', 'https'] and '.' in urllib.parse.urlparse(query).netloc
+    
+    if is_direct_url:
         url = query
+        content_selector = 'body'
     else:
-        search_url_template = SEARCH_ENGINES.get(search_engine)
-        if not search_url_template:
-            return {"status": "error", "query": query, "error_message": f"Invalid search engine: '{search_engine}'."}
-        url = search_url_template.format(query=urllib.parse.quote_plus(query))
+        engine_data = SEARCH_ENGINES.get(search_engine_name)
+        if not engine_data:
+            return {"status": "error", "query": query, "error_message": f"Invalid search engine: '{search_engine_name}'."}
+        url, content_selector = engine_data
+        url = url.format(query=urllib.parse.quote_plus(query))
 
     proxy_config = REVOLVER.get_next()
     proxy_server_used = proxy_config["server"] if proxy_config else "Direct Connection"
     
-    context_args = {
-        'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
-        'java_script_enabled': True, 'ignore_https_errors': True, 'bypass_csp': True
-    }
-    if proxy_config:
-        context_args['proxy'] = proxy_config
+    context_args = {'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', 'java_script_enabled': True, 'ignore_https_errors': True, 'bypass_csp': True, 'accept_downloads': False}
+    if proxy_config: context_args['proxy'] = proxy_config
 
     context = await browser_instance.new_context(**context_args)
     page = await context.new_page()
 
     try:
-        response = await page.goto(url, wait_until='domcontentloaded', timeout=25000)
-        final_url = page.url
+        response = await page.goto(url, wait_until='commit', timeout=20000)
+
+        await asyncio.wait([
+            page.wait_for_load_state('domcontentloaded', timeout=15000),
+            page.wait_for_selector(content_selector, timeout=15000),
+        ], return_when=asyncio.FIRST_COMPLETED)
         
         html_content = await page.content()
+
+        if any(phrase in html_content for phrase in ["unusual traffic", "CAPTCHA", "prove you are human", "before you continue"]):
+            raise Exception("Anti-bot measure detected. Try another search engine or proxy.")
+
+        final_url, title = page.url, await page.title() or "No Title"
         soup = BeautifulSoup(html_content, 'lxml')
         converter = HTML_TO_MARKDOWN_CONVERTER(soup, base_url=final_url)
         markdown_text = converter.convert()
-
-        # HYBRID STRATEGY: If content is empty/trivial, wait briefly for JS to render.
-        if len(markdown_text.split()) < 20:
-            await page.wait_for_timeout(3000)
-            html_content = await page.content()
-            soup = BeautifulSoup(html_content, 'lxml')
-            converter = HTML_TO_MARKDOWN_CONVERTER(soup, base_url=final_url)
-            markdown_text = converter.convert()
-
-        title = await page.title() or "No Title"
         status_code = response.status if response else 0
 
-        return {
-            "status": "success", "query": query, "final_url": final_url, "page_title": title,
-            "http_status": status_code, "proxy_used": proxy_server_used, "markdown_content": markdown_text,
-        }
-    except PlaywrightTimeoutError:
-        return {"status": "error", "query": query, "proxy_used": proxy_server_used, "error_message": f"Navigation timed out after 25s. Site is likely too slow or blocking requests."}
+        return {"status": "success", "query": query, "final_url": final_url, "page_title": title, "http_status": status_code, "proxy_used": proxy_server_used, "markdown_content": markdown_text}
     except Exception as e:
-        return {"status": "error", "query": query, "proxy_used": proxy_server_used, "error_message": str(e).splitlines()[0]}
+        error_message = str(e).splitlines()[0]
+        if "Timeout" in error_message:
+            return {"status": "error", "query": query, "proxy_used": proxy_server_used, "error_message": f"Page failed to load or find content for '{query}'. The site may be slow, blocking automation, or the content selector '{content_selector}' was not found."}
+        return {"status": "error", "query": query, "proxy_used": proxy_server_used, "error_message": error_message}
     finally:
         if 'page' in locals() and not page.is_closed(): await page.close()
         if 'context' in locals(): await context.close()