Repository: apify/crawlee-python
File: src/crawlee/beautifulsoup_crawler/_utils.py (Python)

Code before:
from __future__ import annotations

import re
from typing import TYPE_CHECKING

from bs4 import BeautifulSoup, NavigableString, PageElement, Tag

from crawlee._utils.html_to_text import (
    _ANY_CONSECUTIVE_WHITE_SPACES,
    _EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE,
    _EMPTY_OR_ENDS_WITH_NEW_LINE,
    BLOCK_TAGS,
    SKIP_TAGS,
)

if TYPE_CHECKING:
    from collections.abc import Iterable


def html_to_text(source: str | BeautifulSoup) -> str:
    """Converts markup string or `BeautifulSoup` to newline separated plain text without tags using BeautifulSoup."""
    if isinstance(source, str):
        soup = BeautifulSoup(source)
    elif isinstance(source, BeautifulSoup):
        soup = source
    else:
        raise TypeError('Source must be either a string or a `BeautifulSoup` object.')

    text = ''

    def _page_element_to_text(page_elements: Iterable[PageElement]) -> None:
        """Custom html parsing that performs as implementation from Javascript version of Crawlee."""
        nonlocal text
        for page_element in page_elements:
            if isinstance(page_element, (Tag, NavigableString)):
                if isinstance(page_element, NavigableString):
                    compr: str
                    if isinstance(page_element.parent, Tag) and page_element.parent.name.lower() == 'pre':
                        compr = page_element.get_text()
                    else:
                        # Compress white spaces outside of pre block
                        compr = re.sub(_ANY_CONSECUTIVE_WHITE_SPACES, ' ', page_element.get_text())
                    # If text is empty or ends with a whitespace, don't add the leading whitespace or new line
                    if (compr.startswith((' ', '\n'))) and re.search(_EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE, text):
                        compr = compr[1:]
                    text += compr

Review comment (suggestion):
I believe Honza will want to expose this as a normal function as well. In that case, it will be great if you will document the args and return type as well and expose it to the API docs. (Same for the second variant.)

Code after:
from __future__ import annotations

import re
from typing import TYPE_CHECKING

from bs4 import BeautifulSoup, NavigableString, PageElement, Tag

from crawlee._utils.html_to_text import (
    _ANY_CONSECUTIVE_WHITE_SPACES,
    _EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE,
    _EMPTY_OR_ENDS_WITH_NEW_LINE,
    BLOCK_TAGS,
    SKIP_TAGS,
)

if TYPE_CHECKING:
    from collections.abc import Iterable


def html_to_text(source: str | BeautifulSoup) -> str:
    """Converts markup string or `BeautifulSoup` to newline separated plain text without tags using BeautifulSoup.

    Args:
        source: Input markup string or `BeautifulSoup` object.

    Returns:
        Newline separated plain text without tags.
    """
    if isinstance(source, str):
        soup = BeautifulSoup(source)
    elif isinstance(source, BeautifulSoup):
        soup = source
    else:
        raise TypeError('Source must be either a string or a `BeautifulSoup` object.')

    text = ''

    def _page_element_to_text(page_elements: Iterable[PageElement]) -> None:
        """Custom html parsing that performs as implementation from Javascript version of Crawlee."""
        nonlocal text
        for page_element in page_elements:
            if isinstance(page_element, (Tag, NavigableString)):
                if isinstance(page_element, NavigableString):
                    compr: str
                    if isinstance(page_element.parent, Tag) and page_element.parent.name.lower() == 'pre':
                        compr = page_element.get_text()