Repository: google/budou

Issue:
Span, Zero-width space, or wbr elements?
I just learned this nice work.   It is particularly useful for dyslexia people.

In a meeting of the Japanese DAISY project for textbooks, we discussed how hints for line breaking should be represented.  The use of span elements was suggested.  But people do not want to use span elements for this purpose, because DAISY textbooks already too heavily use span elements for multi-media synchronization.  Thus, Keio Advanced Publishing Laboratory is inclined to adopt the zero-width space or wbr elements.  [Florian's personal draft](https://specs.rivoal.net/css-space-expansion/) is based on this assumption.  See https://github.com/w3c/jlreq/issues/17



Discussion:

Patch:
diff --git a/budou/budou.py b/budou/budou.py
index 7576020..de672b4 100644
--- a/budou/budou.py
+++ b/budou/budou.py
@@ -17,7 +17,7 @@
 """Budou: an automatic organizer tool for beautiful line breaking in CJK
 
 Usage:
-  budou [--segmenter=<seg>] [--language=<lang>] [--classname=<class>] [--inlinestyle] <source>
+  budou [--segmenter=<seg>] [--language=<lang>] [--classname=<class>] [--inlinestyle] [--wbr] <source>
   budou -h | --help
   budou -v | --version
 
@@ -36,6 +36,9 @@ Options:
 
   --inlinestyle               Add :code:`display:inline-block` as inline style
                               attribute.
+
+  --wbr                       User WBR tag for serialization instead of
+                              inline-block SPAN tags.
 """
 
 from __future__ import print_function
@@ -62,12 +65,14 @@ def main():
       language=args['--language'],
       classname=args['--classname'],
       inlinestyle=args['--inlinestyle'],
+      wbr=args['--wbr'],
       )
   print(result['html_code'])
   sys.exit()
 
 def parse(source, segmenter='nlapi', language=None, max_length=None,
-          classname=None, attributes=None, inlinestyle=False, **kwargs):
+          classname=None, attributes=None, inlinestyle=False, wbr=False,
+          **kwargs):
   """Parses input source.
 
   Args:
@@ -79,6 +84,7 @@ def parse(source, segmenter='nlapi', language=None, max_length=None,
     attributes (dict, optional): Attributes for output SPAN tags.
     inlinestyle (bool, optional): Add :code:`display:inline-block` as inline
                                   style attribute.
+    wbr (bool, optional): User WBR tag for serialization.
 
   Returns:
     Results in a dict. :code:`chunks` holds a list of chunks
@@ -88,7 +94,7 @@ def parse(source, segmenter='nlapi', language=None, max_length=None,
   parser = get_parser(segmenter, **kwargs)
   return parser.parse(
       source, language=language, max_length=max_length, classname=classname,
-      attributes=attributes, inlinestyle=inlinestyle)
+      attributes=attributes, inlinestyle=inlinestyle, wbr=wbr)
 
 def authenticate(json_path=None):
   """Gets a Natural Language API parser by authenticating the API.
diff --git a/budou/chunk.py b/budou/chunk.py
index 72c1ff3..cf1f909 100644
--- a/budou/chunk.py
+++ b/budou/chunk.py
@@ -21,6 +21,10 @@ import collections
 from xml.etree import ElementTree as ET
 import unicodedata
 import html5lib
+from html5lib import getTreeWalker
+from html5lib.filters import sanitizer
+from html5lib.constants import namespaces
+
 
 class Chunk:
   """A unit for word segmentation.
@@ -271,7 +275,24 @@ class ChunkList(collections.MutableSequence):
         target_chunks.append(chunk)
     self.list = target_chunks
 
-  def html_serialize(self, attributes, max_length=None):
+  def html_serialize(self, attributes, max_length=None, use_wbr=False):
+    """Returns concatenated HTML code with SPAN tag.
+
+    Args:
+      attributes (dict): A map of name-value pairs for attributes of output
+          SPAN tags.
+      max_length (int, optional): Maximum length of span enclosed chunk.
+      use_wbr (bool, optional): Use WBR tag to serialize the output.
+
+    Returns:
+      The organized HTML code. (str)
+    """
+    if use_wbr:
+      return self.wbr_serialize(max_length)
+    else:
+      return self.span_serialize(attributes, max_length)
+
+  def span_serialize(self, attributes, max_length=None):
     """Returns concatenated HTML code with SPAN tag.
 
     Args:
@@ -309,3 +330,46 @@ class ChunkList(collections.MutableSequence):
         html5lib.parseFragment(result), sanitize=True,
         quote_attr_values='always')
     return result
+
+  def wbr_serialize(self):
+    """Returns concatenated HTML code with WBR tag. This is still experimental.
+
+    Returns:
+      The organized HTML code. (str)
+    """
+    doc = ET.Element('span')
+    doc.attrib['style'] = 'word-break: keep-all'
+    for chunk in self:
+      if (chunk.has_cjk() and doc.text):
+        ele = ET.Element('wbr')
+        doc.append(ele)
+        doc.getchildren()[-1].tail = chunk.word
+      else:
+        # add word without span tag for non-CJK text (e.g. English)
+        # by appending it after the last element
+        if doc.getchildren():
+          if doc.getchildren()[-1].tail is None:
+            doc.getchildren()[-1].tail = chunk.word
+          else:
+            doc.getchildren()[-1].tail += chunk.word
+        else:
+          if doc.text is None:
+            doc.text = chunk.word
+          else:
+            doc.text += chunk.word
+    content = ET.tostring(doc, encoding='utf-8').decode('utf-8')
+    dom = html5lib.parseFragment(content)
+    treewalker = getTreeWalker('etree')
+    stream = treewalker(dom)
+    serializer = html5lib.serializer.HTMLSerializer(
+            quote_attr_values='always')
+    allowed_elements = set(sanitizer.allowed_elements)
+    allowed_elements.add((namespaces['html'], 'wbr'))
+    allowed_css_properties = set(sanitizer.allowed_css_properties)
+    allowed_css_properties.add('word-break')
+    result = serializer.render(sanitizer.Filter(
+        stream, allowed_elements=allowed_elements,
+        allowed_css_properties=allowed_css_properties,
+        ))
+    return result
+
diff --git a/budou/nlapisegmenter.py b/budou/nlapisegmenter.py
index de0bd78..93c98bf 100644
--- a/budou/nlapisegmenter.py
+++ b/budou/nlapisegmenter.py
@@ -171,7 +171,7 @@ class NLAPISegmenter(Segmenter):
       ValueError: If :code:`language` is given and it is not included in
                   :code:`supported_languages`.
     """
-    if language and not language in self.supported_languages:
+    if language and language not in self.supported_languages:
       raise ValueError(
           'Language {} is not supported by NLAPI segmenter'.format(language))
 
diff --git a/budou/parser.py b/budou/parser.py
index a8fa147..753f372 100644
--- a/budou/parser.py
+++ b/budou/parser.py
@@ -57,7 +57,7 @@ class Parser:
     self.segmenter = None
 
   def parse(self, source, language=None, classname=None, max_length=None,
-            attributes=None, inlinestyle=False):
+            attributes=None, inlinestyle=False, wbr=False):
     """Parses the source sentence to output organized HTML code.
 
     Args:
@@ -67,6 +67,7 @@ class Parser:
       attributes (dict, optional): Attributes for output SPAN tags.
       inlinestyle (bool, optional): Add :code:`display:inline-block` as inline
                                     style attribute.
+      wbr (bool, optional): User WBR tag for serialization.
 
     Returns:
       A dictionary containing :code:`chunks` (:obj:`budou.chunk.ChunkList`)
@@ -75,7 +76,8 @@ class Parser:
     attributes = parse_attributes(attributes, classname, inlinestyle)
     source = preprocess(source)
     chunks = self.segmenter.segment(source, language)
-    html_code = chunks.html_serialize(attributes, max_length=max_length)
+    html_code = chunks.html_serialize(
+            attributes, max_length=max_length, use_wbr=wbr)
     return {
         'chunks': chunks,
         'html_code': html_code,
diff --git a/poc.py b/poc.py
new file mode 100644
index 0000000..a167f67
--- /dev/null
+++ b/poc.py
@@ -0,0 +1,11 @@
+import html5lib
+from html5lib import getTreeWalker
+from html5lib.filters import sanitizer
+from html5lib.constants import namespaces, prefixes
+content = '<span>This is <wbr> not a cat</span>'
+dom = html5lib.parseFragment(content)
+treewalker = getTreeWalker('etree')
+serializer = html5lib.serializer.HTMLSerializer()
+stream = treewalker(dom)
+output = serializer.render(sanitizer.Filter(stream, allowed_elements=set([(namespaces['html'], 'span')])))
+print(output)


Test patch:
diff --git a/tests/test_chunk.py b/tests/test_chunk.py
index 0163f9a..5754101 100644
--- a/tests/test_chunk.py
+++ b/tests/test_chunk.py
@@ -114,7 +114,7 @@ class TestChunkList(unittest.TestCase):
         [u'これが', '\n', 'Android'], [chunk.word for chunk in chunks],
         'Trailing spaces in CJK chunk should be converted to breaklines.')
 
-  def test_html_serialize(self):
+  def test_span_serialize(self):
     chunks = ChunkList(Chunk('Hello'), Chunk.space(), Chunk(u'今天'),
         Chunk(u'天气'), Chunk(u'很好'))
     attributes = {
@@ -127,7 +127,7 @@ class TestChunkList(unittest.TestCase):
         u'<span class="foo">天气</span>'
         u'<span class="foo">很好</span>'
         '</span>')
-    result = chunks.html_serialize(attributes, None)
+    result = chunks.span_serialize(attributes, None)
     self.assertEqual(
         result, expected,
         'The chunks should be compiled to a HTML code.')
@@ -140,7 +140,7 @@ class TestChunkList(unittest.TestCase):
     expected = ('<span>'
         'Hey&lt;&lt;script&gt;alert(1)&lt;/script&gt;&gt;guys'
         '</span>')
-    result = chunks.html_serialize(attributes, None)
+    result = chunks.span_serialize(attributes, None)
     self.assertEqual(
         result, expected, 'HTML tags included in a chunk should be encoded.')
 
@@ -155,7 +155,7 @@ class TestChunkList(unittest.TestCase):
         u'インフルエンザに'
         u'<span class="foo">かかった。</span>'
         '</span>')
-    result = chunks.html_serialize(attributes, 6)
+    result = chunks.span_serialize(attributes, 6)
     self.assertEqual(
         result, expected,
         'Chunks that exceed the max length should not be enclosed by a span.')
@@ -163,5 +163,15 @@ class TestChunkList(unittest.TestCase):
   # TODO (tushuhei) Check if TypeError is raised when any instance but Chunk
   # is given to the list.
 
+  def test_wbr_serialize(self):
+    chunks = ChunkList(Chunk(u'今日は'), Chunk(u'ご飯を'), Chunk(u'食べます。'))
+    result = chunks.wbr_serialize()
+    expected = (
+        '<span style="word-break: keep-all;">'
+        u'今日は<wbr></wbr>ご飯を<wbr></wbr>食べます。'
+        '</span>')
+    self.assertEqual(
+        result, expected, 'Chunks should be separated by WBR tags.')
+
 if __name__ == '__main__':
   unittest.main()