Repository: phidatahq/phidata
File: libs/agno/agno/knowledge/reader/web_search_reader.py (Python)

Code before:
        logger.error(f"Failed to fetch content from {url} after {self.max_retries} attempts")
        return None

    def _create_document_from_url(self, url: str, content: str, search_result: Dict[str, str]) -> Document:
        """Create a Document object from URL content and search result metadata"""
        # Use the URL as the document ID
        doc_id = url

        # Use the search result title as the document name, fallback to URL
        doc_name = search_result.get("title", urlparse(url).netloc)

        # Create metadata with search information
        meta_data = {
            "url": url,
            "search_title": search_result.get("title", ""),
            "search_description": search_result.get("description", ""),
            "source": "web_search",
            "search_engine": self.search_engine,
        }

        return Document(id=doc_id, name=doc_name, content=content, meta_data=meta_data)

    def read(self, query: str) -> List[Document]:
        """Read content for a given query by performing web search and fetching content"""
        if not query:
            raise ValueError("Query cannot be empty")

        log_debug(f"Starting web search reader for query: {query}")

        # Perform web search
        search_results = self._perform_web_search(query)
        if not search_results:
            logger.warning(f"No search results found for query: {query}")
            return []

        documents: List[Document] = []

        for result in search_results:
            url = result.get("url", "")

            # Skip if URL is invalid or already visited
            if not self._is_valid_url(url):
                continue

            # Mark URL as visited
            self._visited_urls.add(url)

            # Add delay between requests to be respectful
            if len(documents) > 0:
                time.sleep(self.delay_between_requests)


Review comment (bug):
## Bug 2: WebSearchReader — query 2 skips URLs found by query 1

`_visited_urls` (defined at line 49) is a `Set[str]` meant to deduplicate URLs **within a single query** — so if the same URL appears twice in search results, we don't fetch it twice. But it was never cleared between `read()` calls.

**Before:**
```
reader.read("python tutorials")
  → finds [example.com, docs.python.org]
  → _visited_urls = {example.com, docs.python.org}

reader.read("python advanced")
  → finds [example.com, realpython.com]
  → _is_valid_url("example.com") returns False (already in _visited_urls!)
  → skips example.com even though it's a different query
```

**After:** `_visited_urls.clear()` at the start of each `read()` and `async_read()` ensures each query starts fresh. Within-query dedup still works because URLs are added to the set during the loop at line 232.

Code after:
        logger.error(f"Failed to fetch content from {url} after {self.max_retries} attempts")
        return None

    def _create_document_from_url(self, url: str, content: str, search_result: Dict[str, str]) -> Document:
        """Create a Document object from URL content and search result metadata"""
        # Use the URL as the document ID
        doc_id = url

        # Use the search result title as the document name, fallback to URL
        doc_name = search_result.get("title", urlparse(url).netloc)

        # Create metadata with search information
        meta_data = {
            "url": url,
            "search_title": search_result.get("title", ""),
            "search_description": search_result.get("description", ""),
            "source": "web_search",
            "search_engine": self.search_engine,
        }

        return Document(id=doc_id, name=doc_name, content=content, meta_data=meta_data)

    def read(self, query: str) -> List[Document]:
        """Read content for a given query by performing web search and fetching content"""
        # Clear so URLs from previous queries aren't incorrectly skipped
        self._visited_urls.clear()

        if not query:
            raise ValueError("Query cannot be empty")

        log_debug(f"Starting web search reader for query: {query}")

        # Perform web search
        search_results = self._perform_web_search(query)
        if not search_results:
            logger.warning(f"No search results found for query: {query}")
            return []

        documents: List[Document] = []

        for result in search_results:
            url = result.get("url", "")

            # Skip if URL is invalid or already visited
            if not self._is_valid_url(url):
                continue

            # Mark URL as visited
            self._visited_urls.add(url)

            # Add delay between requests to be respectful