Repository: google/budoux

Issue:
Unopened HTML tag causes exception in budoux 0.6
With budoux 0.6.0

```
budoux --html "foo</p>"

Traceback (most recent call last):
  File "/home/johnc/.local/bin/budoux", line 8, in <module>
    sys.exit(main())
             ^^^^^^
  File "/home/johnc/.local/pipx/venvs/budoux/lib/python3.11/site-packages/budoux/main.py", line 187, in main
    print(_main(test))
          ^^^^^^^^^^^
  File "/home/johnc/.local/pipx/venvs/budoux/lib/python3.11/site-packages/budoux/main.py", line 171, in _main
    res = parser.translate_html_string(inputs_html)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/johnc/.local/pipx/venvs/budoux/lib/python3.11/site-packages/budoux/parser.py", line 102, in translate_html_string
    return resolve(chunks, html)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/johnc/.local/pipx/venvs/budoux/lib/python3.11/site-packages/budoux/html_processor.py", line 124, in resolve
    resolver.feed(html)
  File "/home/johnc/.pyenv/versions/3.11.3/lib/python3.11/html/parser.py", line 110, in feed
    self.goahead(0)
  File "/home/johnc/.pyenv/versions/3.11.3/lib/python3.11/html/parser.py", line 172, in goahead
    k = self.parse_endtag(i)
        ^^^^^^^^^^^^^^^^^^^^
  File "/home/johnc/.pyenv/versions/3.11.3/lib/python3.11/html/parser.py", line 413, in parse_endtag
    self.handle_endtag(elem)
  File "/home/johnc/.local/pipx/venvs/budoux/lib/python3.11/site-packages/budoux/html_processor.py", line 84, in handle_endtag
    self.to_skip = self.element_stack.get_nowait()
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/johnc/.pyenv/versions/3.11.3/lib/python3.11/queue.py", line 199, in get_nowait
    return self.get(block=False)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/johnc/.pyenv/versions/3.11.3/lib/python3.11/queue.py", line 168, in get
    raise Empty
_queue.Empty
```



Discussion:
Thanks for reporting this. The change is due to the non-breaking markup support we introduced at #251, where we need to track the elements in a queue.

Could you elaborate your specific use case that you want to include a close tag with no corresponding open tag? I acknowledge that we need to improve the error message, but BudouX is intended to work with a valid document fragment.
Thanks. It was actually a content bug to have a missing open tag, but I thought worth reporting since it's a change in behaviour.

I don't need this to be a supported case, though maybe a nicer exception would be useful - otherwise happy for you to close as expected behaviour.
Thanks for your input. Raising a better exepction sounds like a plan.
That said, the element queue's emptiness is not enough to detect if the given HTML string is valid or not. The string like below will work with `budoux==0.6.0` CLI even thought the string is not a valid HTML.

```
budoux --html "あの空を見る限り<img>今日はとてもいい天気ですね。</p>"
```

We may need to check void elements as well to manage the element queue and the document's validity better.
@kojiishi any thoughts on this?
Thanks for reporting this issue!

I think the parser should gracefully handle unpaired close tags. Let me look into this, similar to how browsers handle such case.

For self-closing tags such as `<img>`, I think it's supported, but we don't have tests. I'll look into it too.

Resolving patch:
diff --git a/budoux/html_processor.py b/budoux/html_processor.py
--- a/budoux/html_processor.py
+++ b/budoux/html_processor.py
@@ -29,6 +29,19 @@
   SKIP_NODES: typing.Set[str] = set(json.load(f))
 
 
+class ElementState(object):
+  """Represents the state for an element.
+
+  Attributes:
+    tag (str): The tag name.
+    to_skip (bool): Whether the content should be skipped or not.
+  """
+
+  def __init__(self, tag: str, to_skip: bool) -> None:
+    self.tag = tag
+    self.to_skip = to_skip
+
+
 class TextContentExtractor(HTMLParser):
   """An HTML parser to extract text content.
 
@@ -61,7 +74,7 @@ def __init__(self, chunks: typing.List[str], separator: str):
     self.separator = separator
     self.to_skip = False
     self.scan_index = 0
-    self.element_stack: queue.LifoQueue[bool] = queue.LifoQueue()
+    self.element_stack: queue.LifoQueue[ElementState] = queue.LifoQueue()
 
   def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:
     attr_pairs = []
@@ -71,7 +84,7 @@ def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:
       else:
         attr_pairs.append(' %s="%s"' % (attr[0], attr[1]))
     encoded_attrs = ''.join(attr_pairs)
-    self.element_stack.put(self.to_skip)
+    self.element_stack.put(ElementState(tag, self.to_skip))
     if tag.upper() in SKIP_NODES:
       if not self.to_skip and self.chunks_joined[self.scan_index] == SEP:
         self.scan_index += 1
@@ -81,7 +94,17 @@ def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:
 
   def handle_endtag(self, tag: str) -> None:
     self.output += '</%s>' % (tag)
-    self.to_skip = self.element_stack.get_nowait()
+    while not self.element_stack.empty():
+      state = self.element_stack.get_nowait()
+      if state.tag == tag:
+        self.to_skip = state.to_skip
+        break
+      # If the close tag doesn't match the open tag, remove it and keep looking.
+      # This means that close tags close their corresponding open tags.
+      # e.g., `<span>abc<img>def</span>` or `<p>abc<span>def</p>` are both valid
+      # HTML as per the HTML spec.
+      # Note the HTML "adoption agency algorithm" isn't fully supported.
+      # See https://html.spec.whatwg.org/multipage/parsing.html#an-introduction-to-error-handling-and-strange-cases-in-the-parser
 
   def handle_data(self, data: str) -> None:
     for char in data:


Test patch:
diff --git a/tests/test_html_processor.py b/tests/test_html_processor.py
--- a/tests/test_html_processor.py
+++ b/tests/test_html_processor.py
@@ -46,6 +46,38 @@ def test_output(self) -> None:
     self.assertEqual(resolver.output, expected,
                      'WBR tags should be inserted as specified by chunks.')
 
+  def test_unpaired(self) -> None:
+    input = '<p>abcdef</p></p>'
+    expected = '<p>abc<wbr>def</p></p>'
+    resolver = html_processor.HTMLChunkResolver(['abc', 'def'], '<wbr>')
+    resolver.feed(input)
+    self.assertEqual(resolver.output, expected,
+                     'Unpaired close tag should not cause errors.')
+
+  def test_nobr(self) -> None:
+    input = '<p>ab<nobr>cde</nobr>f</p>'
+    expected = '<p>ab<nobr>cde</nobr>f</p>'
+    resolver = html_processor.HTMLChunkResolver(['abc', 'def'], '<wbr>')
+    resolver.feed(input)
+    self.assertEqual(resolver.output, expected,
+                     'WBR tags should not be inserted if in NOBR.')
+
+  def test_after_nobr(self) -> None:
+    input = '<p>ab<nobr>xy</nobr>abcdef</p>'
+    expected = '<p>ab<nobr>xy</nobr>abc<wbr>def</p>'
+    resolver = html_processor.HTMLChunkResolver(['abxyabc', 'def'], '<wbr>')
+    resolver.feed(input)
+    self.assertEqual(resolver.output, expected,
+                     'WBR tags should be inserted if after NOBR.')
+
+  def test_img_in_nobr(self) -> None:
+    input = '<p>ab<nobr>x<img>y</nobr>abcdef</p>'
+    expected = '<p>ab<nobr>x<img>y</nobr>abc<wbr>def</p>'
+    resolver = html_processor.HTMLChunkResolver(['abxyabc', 'def'], '<wbr>')
+    resolver.feed(input)
+    self.assertEqual(resolver.output, expected,
+                     'IMG should not affect surrounding NOBR.')
+
 
 class TestResolve(unittest.TestCase):