Repository: google/budou

Issue:
not proper grouping of comma  (読点)
<img width="354" alt="default" src="https://cloud.githubusercontent.com/assets/968152/24842452/bcee80fa-1dd2-11e7-848a-86b683f9dab6.png">

in this case, comma  (読点) should be grouped with former word (最終日).

<img width="357" alt="default" src="https://cloud.githubusercontent.com/assets/968152/24843094/48a93066-1dda-11e7-87f8-a684e52ebfc1.png">

<img width="389" alt="default" src="https://cloud.githubusercontent.com/assets/968152/24843125/a27b4160-1dda-11e7-9039-775f1542d099.png">

<img width="356" alt="default" src="https://cloud.githubusercontent.com/assets/968152/24843143/de3f7cf2-1dda-11e7-9ae7-320e84427d12.png">

<img width="364" alt="default" src="https://cloud.githubusercontent.com/assets/968152/24843153/f7d6d426-1dda-11e7-9c2a-82852ebf3d0d.png">


Discussion:

Patch:
diff --git a/budou/budou.py b/budou/budou.py
index da1a28f..f547f61 100644
--- a/budou/budou.py
+++ b/budou/budou.py
@@ -26,6 +26,7 @@ from oauth2client.client import GoogleCredentials
 import oauth2client.service_account
 import re
 import six
+import unicodedata
 
 Chunk = collections.namedtuple('Chunk', ['word', 'pos', 'label', 'forward'])
 """Word chunk object.
@@ -158,12 +159,42 @@ class Budou(object):
       A list of Chunks.
     """
     chunks = self._get_source_chunks(input_text, language)
+    chunks = self._update_punct_direction(chunks)
     for forward in (True, False):
       condition = lambda chunk: (
           chunk.label in TARGET_LABEL or chunk.pos == 'PUNCT')
       chunks = self._concatenate_inner(chunks, condition, forward)
     return chunks
 
+  def _update_punct_direction(self, chunks):
+    """Updates chunk's concatenate direction if it is a punctuation mark.
+
+    Args:
+      chunks: A list of Chunks.
+
+    Returns:
+      A list of updated Chunks.
+    """
+    result = []
+    for chunk in chunks:
+      if chunk.pos == 'PUNCT':
+        forward = False
+        try:
+          # Getting unicode category to determine the direction.
+          # Concatenates to the following if it belongs to Ps or Pi category.
+          # Ps: Punctuation, open (e.g. opening bracket characters)
+          # Pi: Punctuation, initial quote (e.g. opening quotation mark)
+          # Otherwise, concatenates to the previous word.
+          # See also https://en.wikipedia.org/wiki/Unicode_character_property
+          category = unicodedata.category(chunk.word)
+          if category in ('Ps', 'Pi'):
+            forward = True
+        except:
+          pass
+        chunk = Chunk(chunk.word, chunk.pos, chunk.label, forward)
+      result.append(chunk)
+    return result
+
   def _get_attribute_dict(self, attributes, classname=None):
     """Returns a dictionary of attribute name-value pairs.
 


Test patch:
diff --git a/test/budou_test.py b/test/budou_test.py
index 8c3b30e..134487d 100644
--- a/test/budou_test.py
+++ b/test/budou_test.py
@@ -289,6 +289,36 @@ class TestBudouMethods(unittest.TestCase):
         result, expected,
         'Input text should be parsed into chunks separated by spaces.')
 
+  def test_update_punct_direction(self):
+    chunks = [
+        budou.Chunk(u'。', u'PUNCT', None, None),
+        budou.Chunk(u'、', u'PUNCT', None, None),
+        budou.Chunk(u'「', u'PUNCT', None, None),
+        budou.Chunk(u'」', u'PUNCT', None, None),
+        budou.Chunk(u'（', u'PUNCT', None, None),
+        budou.Chunk(u'）', u'PUNCT', None, None),
+        budou.Chunk(u'[', u'PUNCT', None, None),
+        budou.Chunk(u']', u'PUNCT', None, None),
+        budou.Chunk(u'(', u'PUNCT', None, None),
+        budou.Chunk(u')', u'PUNCT', None, None),
+    ]
+    expected = [
+        budou.Chunk(u'。', u'PUNCT', None, False),
+        budou.Chunk(u'、', u'PUNCT', None, False),
+        budou.Chunk(u'「', u'PUNCT', None, True),
+        budou.Chunk(u'」', u'PUNCT', None, False),
+        budou.Chunk(u'（', u'PUNCT', None, True),
+        budou.Chunk(u'）', u'PUNCT', None, False),
+        budou.Chunk(u'[', u'PUNCT', None, True),
+        budou.Chunk(u']', u'PUNCT', None, False),
+        budou.Chunk(u'(', u'PUNCT', None, True),
+        budou.Chunk(u')', u'PUNCT', None, False),
+    ]
+    result = self.parser._update_punct_direction(chunks)
+    self.assertEqual(
+        result, expected,
+        'Punctuation mark\'s concatenating direction should be determined.')
+
 
 if __name__ == '__main__':
   unittest.main()