Repository: apify/crawlee-python
File: src/crawlee/beautifulsoup_crawler/_beautifulsoup_crawler.py (Python)

Code before:
    ) -> AsyncGenerator[BeautifulSoupCrawlingContext, None]:
        """Parse the HTTP response using the `BeautifulSoup` library and implements the `enqueue_links` function.

        Args:
            context: The current crawling context.

        Yields:
            The enhanced crawling context with the `BeautifulSoup` selector and the `enqueue_links` function.
        """
        soup = await asyncio.to_thread(lambda: BeautifulSoup(context.http_response.read(), self._parser))

        async def enqueue_links(
            *,
            selector: str = 'a',
            label: str | None = None,
            user_data: dict[str, Any] | None = None,
            **kwargs: Unpack[AddRequestsKwargs],
        ) -> None:
            kwargs.setdefault('strategy', EnqueueStrategy.SAME_HOSTNAME)

            if self._max_crawl_depth is not None and context.request.crawl_depth + 1 > self._max_crawl_depth:
                context.log.info(
                    f'Skipping enqueue_links for URL "{context.request.url}" due to the maximum crawl depth limit.'
                )
                return

            requests = list[BaseRequestData]()
            user_data = user_data or {}

            link: Tag
            for link in soup.select(selector):
                link_user_data = user_data

                if label is not None:
                    link_user_data.setdefault('label', label)

                data = {'crawlDepth': context.request.crawl_depth + 1}
                link_user_data.setdefault('__crawlee', CrawleeRequestData(**data))

                if (url := link.attrs.get('href')) is not None:
                    url = url.strip()

                    if not is_url_absolute(url):
                        url = convert_to_absolute_url(context.request.url, url)

                    try:
                        request = BaseRequestData.from_url(url, user_data=link_user_data)
                    except ValidationError as exc:
                        context.log.debug(
                            f'Skipping URL "{url}" due to invalid format: {exc}. '
                            'This may be caused by a malformed URL or unsupported URL scheme. '

Review comment (suggestion):
1. the repetition in all crawlers is not great
    - it is very easy to overlook this when implementing a new crawler
3. it is also possible to add requests to the queue via `context.add_requests`
    - ideally we should fill in default depth to the requests - it doesn't make much sense to exempt some requests from `max_crawl_depth`
    - if there is no ergonomic way to support max depth in `add_requests`, we should make it clear in the docs that the user has to handle crawl depth manually



Code after:
    ) -> AsyncGenerator[BeautifulSoupCrawlingContext, None]:
        """Parse the HTTP response using the `BeautifulSoup` library and implements the `enqueue_links` function.

        Args:
            context: The current crawling context.

        Yields:
            The enhanced crawling context with the `BeautifulSoup` selector and the `enqueue_links` function.
        """
        soup = await asyncio.to_thread(lambda: BeautifulSoup(context.http_response.read(), self._parser))

        async def enqueue_links(
            *,
            selector: str = 'a',
            label: str | None = None,
            user_data: dict[str, Any] | None = None,
            **kwargs: Unpack[AddRequestsKwargs],
        ) -> None:
            kwargs.setdefault('strategy', EnqueueStrategy.SAME_HOSTNAME)

            requests = list[BaseRequestData]()
            user_data = user_data or {}

            link: Tag
            for link in soup.select(selector):
                link_user_data = user_data

                if label is not None:
                    link_user_data.setdefault('label', label)

                if (url := link.attrs.get('href')) is not None:
                    url = url.strip()

                    if not is_url_absolute(url):
                        url = convert_to_absolute_url(context.request.url, url)

                    try:
                        request = BaseRequestData.from_url(url, user_data=link_user_data)
                    except ValidationError as exc:
                        context.log.debug(
                            f'Skipping URL "{url}" due to invalid format: {exc}. '
                            'This may be caused by a malformed URL or unsupported URL scheme. '
                            'Please ensure the URL is correct and retry.'
                        )
                        continue

                    requests.append(request)

            await context.add_requests(requests, **kwargs)

        yield BeautifulSoupCrawlingContext(