Repository: deepset-ai/haystack

Issue:
SerperDevWebSearch: Add exclude_subdomains parameter

### Description
`SerperDevWebSearch` component currently includes subdomains when using the `allowed_domains` [parameter](https://github.com/deepset-ai/haystack/blob/main/haystack/components/websearch/serper_dev.py#L46). This can be problematic when users want to limit results to only the main domain.

### Use Case
User wants to search only within `company.com` but exclude results from subdomains like `blog.company.com`, `shop.company.com`, etc.


Discussion:

Patch:
diff --git a/haystack/components/websearch/serper_dev.py b/haystack/components/websearch/serper_dev.py
index be679cf10e..8663df4367 100644
--- a/haystack/components/websearch/serper_dev.py
+++ b/haystack/components/websearch/serper_dev.py
@@ -4,6 +4,7 @@
 
 import json
 from typing import Any, Optional, Union
+from urllib.parse import urlparse
 
 import requests
 
@@ -36,6 +37,15 @@ class SerperDevWebSearch:
 
     assert results["documents"]
     assert results["links"]
+
+    # Example with domain filtering - exclude subdomains
+    websearch_filtered = SerperDevWebSearch(
+        top_k=10,
+        allowed_domains=["example.com"],
+        exclude_subdomains=True,  # Only results from example.com, not blog.example.com
+        api_key=Secret.from_token("test-api-key")
+    )
+    results_filtered = websearch_filtered.run(query="search query")
     ```
     """
 
@@ -45,6 +55,8 @@ def __init__(
         top_k: Optional[int] = 10,
         allowed_domains: Optional[list[str]] = None,
         search_params: Optional[dict[str, Any]] = None,
+        *,
+        exclude_subdomains: bool = False,
     ):
         """
         Initialize the SerperDevWebSearch component.
@@ -52,6 +64,9 @@ def __init__(
         :param api_key: API key for the Serper API.
         :param top_k: Number of documents to return.
         :param allowed_domains: List of domains to limit the search to.
+        :param exclude_subdomains: Whether to exclude subdomains when filtering by allowed_domains.
+            If True, only results from the exact domains in allowed_domains will be returned.
+            If False, results from subdomains will also be included. Defaults to False.
         :param search_params: Additional parameters passed to the Serper API.
             For example, you can set 'num' to 20 to increase the number of search results.
             See the [Serper website](https://serper.dev/) for more details.
@@ -59,6 +74,7 @@ def __init__(
         self.api_key = api_key
         self.top_k = top_k
         self.allowed_domains = allowed_domains
+        self.exclude_subdomains = exclude_subdomains
         self.search_params = search_params or {}
 
         # Ensure that the API key is resolved.
@@ -75,6 +91,7 @@ def to_dict(self) -> dict[str, Any]:
             self,
             top_k=self.top_k,
             allowed_domains=self.allowed_domains,
+            exclude_subdomains=self.exclude_subdomains,
             search_params=self.search_params,
             api_key=self.api_key.to_dict(),
         )
@@ -90,6 +107,36 @@ def from_dict(cls, data: dict[str, Any]) -> "SerperDevWebSearch":
         deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
         return default_from_dict(cls, data)
 
+    def _is_domain_allowed(self, url: str) -> bool:
+        """
+        Check if a URL's domain is allowed based on allowed_domains and exclude_subdomains settings.
+
+        :param url: The URL to check.
+        :returns: True if the domain is allowed, False otherwise.
+        """
+        if not self.allowed_domains:
+            return True
+
+        try:
+            parsed = urlparse(url)
+            domain = parsed.netloc.lower()
+
+            for allowed_domain in self.allowed_domains:
+                allowed_domain = allowed_domain.lower()
+
+                if self.exclude_subdomains:
+                    # Exact domain match only
+                    if domain == allowed_domain:
+                        return True
+                # Allow subdomains (current behavior)
+                elif domain == allowed_domain or domain.endswith("." + allowed_domain):
+                    return True
+
+            return False
+        except Exception:
+            # If URL parsing fails, allow the result to be safe
+            return True
+
     @component.output_types(documents=list[Document], links=list[str])
     def run(self, query: str) -> dict[str, Union[list[Document], list[str]]]:
         """
@@ -125,6 +172,7 @@ def run(self, query: str) -> dict[str, Union[list[Document], list[str]]]:
         organic = [
             Document(meta={k: v for k, v in d.items() if k != "snippet"}, content=d.get("snippet"))
             for d in json_result["organic"]
+            if self._is_domain_allowed(d.get("link", ""))
         ]
 
         # answer box is what search engine shows as a direct answer to the query
@@ -143,7 +191,7 @@ def run(self, query: str) -> dict[str, Union[list[Document], list[str]]]:
                     if key in answer_dict:
                         answer_box_content = answer_dict[key]
                         break
-            if answer_box_content:
+            if answer_box_content and self._is_domain_allowed(answer_dict.get("link", "")):
                 answer_box = [
                     Document(
                         content=answer_box_content,
@@ -155,17 +203,18 @@ def run(self, query: str) -> dict[str, Union[list[Document], list[str]]]:
         people_also_ask = []
         if "peopleAlsoAsk" in json_result:
             for result in json_result["peopleAlsoAsk"]:
-                title = result.get("title", "")
-                people_also_ask.append(
-                    Document(
-                        content=result["snippet"] if result.get("snippet") else title,
-                        meta={"title": title, "link": result.get("link", None)},
+                if self._is_domain_allowed(result.get("link", "")):
+                    title = result.get("title", "")
+                    people_also_ask.append(
+                        Document(
+                            content=result["snippet"] if result.get("snippet") else title,
+                            meta={"title": title, "link": result.get("link", None)},
+                        )
                     )
-                )
 
         documents = answer_box + organic + people_also_ask
 
-        links = [result["link"] for result in json_result["organic"]]
+        links = [result["link"] for result in json_result["organic"] if self._is_domain_allowed(result.get("link", ""))]
 
         logger.debug(
             "Serper Dev returned {number_documents} documents for the query '{query}'",
diff --git a/releasenotes/notes/serperdev-add-exclude-subdomains-param-932b8fe4a001f378.yaml b/releasenotes/notes/serperdev-add-exclude-subdomains-param-932b8fe4a001f378.yaml
new file mode 100644
index 0000000000..423d6e2857
--- /dev/null
+++ b/releasenotes/notes/serperdev-add-exclude-subdomains-param-932b8fe4a001f378.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add `exclude_subdomains` parameter to `SerperDevWebSearch` component. When set to `True`, this parameter
+    restricts search results to only the exact domains specified in `allowed_domains`, excluding any subdomains.
+    For example, with `allowed_domains=["example.com"]` and `exclude_subdomains=True`, results from
+    "blog.example.com" or "shop.example.com" will be filtered out, returning only results from "example.com".
+    The parameter defaults to `False` to maintain backward compatibility with existing behavior.


Test patch:
diff --git a/test/components/websearch/test_serperdev.py b/test/components/websearch/test_serperdev.py
index 92efc43bc6..e797e91e7f 100644
--- a/test/components/websearch/test_serperdev.py
+++ b/test/components/websearch/test_serperdev.py
@@ -146,6 +146,7 @@ def test_to_dict(self, monkeypatch):
                 "api_key": {"env_vars": ["SERPERDEV_API_KEY"], "strict": True, "type": "env_var"},
                 "top_k": 10,
                 "allowed_domains": ["test.com"],
+                "exclude_subdomains": False,
                 "search_params": {"param": "test"},
             },
         }
@@ -202,6 +203,109 @@ def test_web_search(self):
         documents = results["documents"]
         links = results["links"]
         assert len(documents) == len(links) == 10
-        assert all(isinstance(doc, Document) for doc in results)
+        assert all(isinstance(doc, Document) for doc in documents)
         assert all(isinstance(link, str) for link in links)
         assert all(link.startswith("http") for link in links)
+
+    def test_exclude_subdomains_filtering(self, monkeypatch):
+        """Test that exclude_subdomains parameter properly filters results."""
+        monkeypatch.setenv("SERPERDEV_API_KEY", "test-api-key")
+
+        # Mock response with mixed domains and subdomains
+        mock_response = {
+            "organic": [
+                {
+                    "title": "Main domain result",
+                    "link": "https://example.com/page1",
+                    "snippet": "Content from main domain",
+                },
+                {
+                    "title": "Subdomain result 1",
+                    "link": "https://blog.example.com/post1",
+                    "snippet": "Content from blog subdomain",
+                },
+                {
+                    "title": "Subdomain result 2",
+                    "link": "https://shop.example.com/product1",
+                    "snippet": "Content from shop subdomain",
+                },
+                {
+                    "title": "Different domain result",
+                    "link": "https://other.com/page1",
+                    "snippet": "Content from different domain",
+                },
+            ]
+        }
+
+        with patch("haystack.components.websearch.serper_dev.requests") as mock_requests:
+            mock_requests.post.return_value = Mock(status_code=200, json=lambda: mock_response)
+
+            # Test with exclude_subdomains=False (default behavior)
+            ws_include = SerperDevWebSearch(
+                api_key=Secret.from_token("test-api-key"), allowed_domains=["example.com"], exclude_subdomains=False
+            )
+            results_include = ws_include.run(query="test query")
+
+            # Should include main domain and subdomains but exclude other domains
+            assert len(results_include["documents"]) == 3  # example.com + 2 subdomains
+            assert len(results_include["links"]) == 3
+
+            included_links = results_include["links"]
+            assert "https://example.com/page1" in included_links
+            assert "https://blog.example.com/post1" in included_links
+            assert "https://shop.example.com/product1" in included_links
+            assert "https://other.com/page1" not in included_links
+
+            # Test with exclude_subdomains=True
+            ws_exclude = SerperDevWebSearch(
+                api_key=Secret.from_token("test-api-key"), allowed_domains=["example.com"], exclude_subdomains=True
+            )
+            results_exclude = ws_exclude.run(query="test query")
+
+            # Should only include main domain, exclude subdomains and other domains
+            assert len(results_exclude["documents"]) == 1  # only example.com
+            assert len(results_exclude["links"]) == 1
+
+            excluded_links = results_exclude["links"]
+            assert "https://example.com/page1" in excluded_links
+            assert "https://blog.example.com/post1" not in excluded_links
+            assert "https://shop.example.com/product1" not in excluded_links
+            assert "https://other.com/page1" not in excluded_links
+
+    def test_is_domain_allowed_helper_method(self, monkeypatch):
+        """Test the _is_domain_allowed helper method directly."""
+        monkeypatch.setenv("SERPERDEV_API_KEY", "test-api-key")
+
+        # Test with exclude_subdomains=False
+        ws_include = SerperDevWebSearch(
+            api_key=Secret.from_token("test-api-key"),
+            allowed_domains=["example.com", "test.org"],
+            exclude_subdomains=False,
+        )
+
+        # Should allow main domains and subdomains
+        assert ws_include._is_domain_allowed("https://example.com/page") == True
+        assert ws_include._is_domain_allowed("https://blog.example.com/post") == True
+        assert ws_include._is_domain_allowed("https://shop.example.com/product") == True
+        assert ws_include._is_domain_allowed("https://test.org/page") == True
+        assert ws_include._is_domain_allowed("https://sub.test.org/page") == True
+        assert ws_include._is_domain_allowed("https://other.com/page") == False
+
+        # Test with exclude_subdomains=True
+        ws_exclude = SerperDevWebSearch(
+            api_key=Secret.from_token("test-api-key"),
+            allowed_domains=["example.com", "test.org"],
+            exclude_subdomains=True,
+        )
+
+        # Should only allow exact domain matches
+        assert ws_exclude._is_domain_allowed("https://example.com/page") == True
+        assert ws_exclude._is_domain_allowed("https://blog.example.com/post") == False
+        assert ws_exclude._is_domain_allowed("https://shop.example.com/product") == False
+        assert ws_exclude._is_domain_allowed("https://test.org/page") == True
+        assert ws_exclude._is_domain_allowed("https://sub.test.org/page") == False
+        assert ws_exclude._is_domain_allowed("https://other.com/page") == False
+
+        # Test with no allowed_domains (should allow all)
+        ws_no_filter = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"), allowed_domains=None)
+        assert ws_no_filter._is_domain_allowed("https://any.domain.com/page") == True