Repository: oppia/oppia
File: core/domain/voiceover_regeneration_services.py (Python)

Code before:
    """
    tag_name = element.name
    voiceover_text_extractor_fn = (
        CUSTOM_RTE_TAGS_TO_VOICEOVER_TEXT_EXTRACTION_RULES.get(
            tag_name, _return_empty_string))

    element.string = voiceover_text_extractor_fn(element)
    element.name = 'p'
    return element


def parse_html(html_content: str) -> str:
    """The method processes the HTML content and extracts the plain text.

    Args:
        html_content: str. HTML content that will be transformed into plain
            text.

    Returns:
        str. The plain text retrieved from the HTML content.

    Raises:
        Exception. The HTML content contains invalid or unsupported tags.
    """
    soup = bs4.BeautifulSoup(html_content, 'html.parser')
    all_tags = {tag.name for tag in soup.find_all()}

    allowed_custom_oppia_rte_tags = list(
        rte_component_registry.Registry.get_tag_list_with_attrs().keys())
    allowed_general_oppia_rte_tags = list(html_cleaner.ATTRS_ALLOWLIST.keys())

    unknown_tags = (
        all_tags -
        set(allowed_custom_oppia_rte_tags) -
        set(allowed_general_oppia_rte_tags)
    )

    if unknown_tags:
        raise Exception(
            'HTML content contains invalid or unsupported tags: %s' % (
                ', '.join(unknown_tags)))

    for custom_tag_element in allowed_custom_oppia_rte_tags:
        for element in soup.find_all(custom_tag_element):
            convert_custom_oppia_tags_to_generic_tags(element)

    text_content: str = get_text_with_delimiters(
        soup, delimiter=feconf.OPPIA_CONTENT_TAG_DELIMITER)

    return text_content


Review comment (suggestion):
I would separate the validity check into a separate function (with its own tests) and call it from here.

Though, do we not already have something that can handle this in html_cleaner? And also, to confirm, we don't have any validation checks anywhere for the html in the voiceover pipeline atm? If so, I agree that it's a good idea to add one.

Code after:
    """
    tag_name = element.name
    voiceover_text_extractor_fn = (
        CUSTOM_RTE_TAGS_TO_VOICEOVER_TEXT_EXTRACTION_RULES.get(
            tag_name, _return_empty_string))

    element.string = voiceover_text_extractor_fn(element)
    element.name = 'p'
    return element


def parse_html(html_content: str) -> str:
    """The method processes the HTML content and extracts the plain text.

    Args:
        html_content: str. HTML content that will be transformed into plain
            text.

    Returns:
        str. The plain text retrieved from the HTML content.

    Raises:
        Exception. The HTML content contains invalid or unsupported tags.
    """
    html_cleaner.validate_rte_tags(html_content)
    soup = bs4.BeautifulSoup(html_content, 'html.parser')

    allowed_custom_oppia_rte_tags = list(
        rte_component_registry.Registry.get_tag_list_with_attrs().keys())

    for custom_tag_element in allowed_custom_oppia_rte_tags:
        for element in soup.find_all(custom_tag_element):
            convert_custom_oppia_tags_to_generic_tags(element)

    text_content: str = get_text_with_delimiters(
        soup, delimiter=feconf.OPPIA_CONTENT_TAG_DELIMITER)

    return text_content


def get_text_with_delimiters(soup: bs4.BeautifulSoup, delimiter: str) -> str:
    """The method extracts text from the BeautifulSoup object and
    adds delimiters between text segments based on the block-level HTML tags.

    Args:
        soup: BeautifulSoup. The BeautifulSoup object containing the HTML
            content.
        delimiter: str. The delimiter to be added between text segments.

    Returns:
        str. The text content with delimiters added between segments.