Repository: dmlc/gluon-nlp

Issue:
Better tokenizer for Chinese
Currently, we support the MosesTokenizer in NLTK and the spaCy tokenizer. Both are not very good for tokenizing Chinese sentences. We could consider adding some Chinese tokenizers.

I've searched online and found these two candidates (There would be more):
1) Standford segmenter: https://nlp.stanford.edu/software/segmenter.html , http://www.nltk.org/_modules/nltk/tokenize/stanford_segmenter.html
2) Jieba tokenizer: https://github.com/fxsjy/jieba

We can add them in [transforms.py](https://github.com/dmlc/gluon-nlp/blob/master/gluonnlp/data/transforms.py).


Discussion:
Can we create a tokenizer class only for Chinese here? Or still designing a general class (like **NLTKMosesTokenizer**) and setting the default "lang" as zh?
I think it’s a good solution to create a tokenizer class only for Chinese.

Get Outlook for iOS<https://aka.ms/o0ukef>
________________________________
From: LI XIN <notifications@github.com>
Sent: Saturday, June 16, 2018 5:33:24 PM
To: dmlc/gluon-nlp
Cc: Xingjian SHI; Author
Subject: Re: [dmlc/gluon-nlp] Better tokenizer for Chinese (#102)


Can we create a tokenizer class only for Chinese here? Or still designing a general class (like NLTKMosesTokenizer) and setting the default "lang" as zh?

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub<https://github.com/dmlc/gluon-nlp/issues/102#issuecomment-397800615>, or mute the thread<https://github.com/notifications/unsubscribe-auth/AE8D7giVaJ3ayGkFqO-CuLXps526l_zRks5t9NDkgaJpZM4T7Beo>.


Resolving patch:
diff --git a/gluonnlp/data/transforms.py b/gluonnlp/data/transforms.py
--- a/gluonnlp/data/transforms.py
+++ b/gluonnlp/data/transforms.py
@@ -24,10 +24,14 @@
 from __future__ import print_function
 
 __all__ = ['ClipSequence', 'PadSequence', 'NLTKMosesTokenizer', 'SpacyTokenizer',
-           'NLTKMosesDetokenizer']
+           'NLTKMosesDetokenizer', 'JiebaTokenizer', 'NLTKStanfordSegmenter']
+
+import os
 
 import numpy as np
 import mxnet as mx
+from mxnet.gluon.utils import download, check_sha1
+from .utils import _get_home_dir, _extract_archive
 
 
 class ClipSequence(object):
@@ -326,3 +330,171 @@ def __call__(self, sample, return_str=False):
             List of words or detokenized text
         """
         return self._detokenizer.detokenize(sample, return_str=return_str)
+
+
+class JiebaTokenizer(object):
+    r"""Apply the jieba Tokenizer.
+
+    Users of this class are required to `install jieba <https://github.com/fxsjy/jieba>`_
+
+    Parameters
+    ----------
+    lang : str
+        The language to tokenize. Default is "zh", i.e, Chinese.
+
+    Examples
+    --------
+    >>> tokenizer = JiebaTokenizer()
+    >>> tokenizer(u"我来到北京清华大学")
+    ['我',
+     '来到',
+     '北京',
+     '清华大学']
+    >>> tokenizer(u"小明硕士毕业于中国科学院计算所，后在日本京都大学深造")
+    ['小明',
+     '硕士',
+     '毕业',
+     '于',
+     '中国科学院',
+     '计算所',
+     '，',
+     '后',
+     '在',
+     '日本京都大学',
+     '深造']
+
+    """
+    def __init__(self):
+        try:
+            import jieba
+        except ImportError:
+            raise ImportError('jieba is not installed. You must install jieba in order to use the '
+                              'JiebaTokenizer. You can refer to the official installation guide '
+                              'in https://github.com/fxsjy/jieba')
+        self._tokenizer = jieba
+
+    def __call__(self, sample):
+        """
+
+        Parameters
+        ----------
+        sample: str
+            The Chinese sentence to tokenize. Better not to input sentence in other languages
+            since this class is mainly used for Chinese Word Segmentation.
+
+        Returns
+        -------
+        ret : list of strs
+            List of tokens
+        """
+        # we use default cutting mode provided by jieba, i.e., accurate mode
+        return [tok for tok in self._tokenizer.cut(sample) if tok != ' ' and tok != '']
+
+
+class NLTKStanfordSegmenter(object):
+    r"""Apply the Stanford Chinese Word Segmenter implemented in NLTK.
+
+    Users of this class are required to install Java, NLTK and download Stanford Word Segmenter
+
+    Parameters
+    ----------
+    segmenter_root : str, default '$MXNET_HOME/stanford-segmenter'
+        Path to folder for storing stanford segmenter.
+        MXNET_HOME defaults to '~/.mxnet'.
+
+    slf4j_root : str, default '$MXNET_HOME/slf4j'
+        Path to foler for storing slf4j.
+        MXNET_HOME defaults to '~/.mxnet'
+
+    java_class : str, default 'edu.stanford.nlp.ie.crf.CRFClassifier'
+        The learning algorithm used for segmentation
+
+    Examples
+    --------
+    >>> tokenizer = NLTKStanfordSegmenter()
+    >>> tokenizer(u"我来到北京清华大学")
+    ['我',
+     '来到',
+     '北京',
+     '清华大学']
+    >>> tokenizer(u"小明硕士毕业于中国科学院计算所，后在日本京都大学深造")
+    ['小明',
+     '硕士',
+     '毕业',
+     '于',
+     '中国',
+     '科学院',
+     '计算所',
+     '，',
+     '后',
+     '在',
+     '日本'
+     '京都大学',
+     '深造']
+
+    """
+    def __init__(self, segmenter_root=os.path.join(_get_home_dir(), 'stanford-segmenter'),
+                 slf4j_root=os.path.join(_get_home_dir(), 'slf4j'),
+                 java_class='edu.stanford.nlp.ie.crf.CRFClassifier'):
+        is_java_exist = os.system('java -version')
+        assert is_java_exist == 0, 'Java is not installed. You must install Java 8.0' \
+                                   'in order to use the NLTKStanfordSegmenter'
+        try:
+            from nltk.tokenize import StanfordSegmenter
+        except ImportError:
+            raise ImportError('NLTK or relevant packages are not installed. You must install NLTK '
+                              'in order to use the NLTKStanfordSegmenter. You can refer to the '
+                              'official installation guide in https://www.nltk.org/install.html.')
+        path_to_jar = os.path.join(segmenter_root, 'stanford-segmenter-2018-02-27',
+                                   'stanford-segmenter-3.9.1.jar')
+        path_to_model = os.path.join(segmenter_root, 'stanford-segmenter-2018-02-27',
+                                     'data', 'pku.gz')
+        path_to_dict = os.path.join(segmenter_root, 'stanford-segmenter-2018-02-27',
+                                    'data', 'dict-chris6.ser.gz')
+        path_to_sihan_corpora_dict = os.path.join(segmenter_root,
+                                                  'stanford-segmenter-2018-02-27', 'data')
+        segmenter_url = 'https://nlp.stanford.edu/software/stanford-segmenter-2018-02-27.zip'
+        segmenter_sha1 = 'aa27a6433704b7b4c6a14be1c126cb4b14b8f57b'
+        stanford_segmenter = os.path.join(segmenter_root, 'stanford-segmenter-2018-02-27.zip')
+        if not os.path.exists(path_to_jar) or \
+                not os.path.exists(path_to_model) or \
+                not os.path.exists(path_to_dict) or \
+                not os.path.exists(path_to_sihan_corpora_dict) or \
+                not check_sha1(filename=stanford_segmenter, sha1_hash=segmenter_sha1):
+            # automatically download the files from the website and place them to stanford_root
+            if not os.path.exists(segmenter_root):
+                os.mkdir(segmenter_root)
+            download(url=segmenter_url, path=segmenter_root, sha1_hash=segmenter_sha1)
+            _extract_archive(file=stanford_segmenter, target_dir=segmenter_root)
+
+        path_to_slf4j = os.path.join(slf4j_root, 'slf4j-1.7.25', 'slf4j-api-1.7.25.jar')
+        slf4j_url = 'https://www.slf4j.org/dist/slf4j-1.7.25.zip'
+        slf4j_sha1 = '89ea41ad6ebe1b190139421bb7c8d981e9df1625'
+        slf4j = os.path.join(slf4j_root, 'slf4j-1.7.25.zip')
+        if not os.path.exists(path_to_slf4j) or \
+                not check_sha1(filename=slf4j, sha1_hash=slf4j_sha1):
+            # automatically download the files from the website and place them to slf4j_root
+            if not os.path.exists(slf4j_root):
+                os.mkdir(slf4j_root)
+            download(url=slf4j_url, path=slf4j_root, sha1_hash=slf4j_sha1)
+            _extract_archive(file=slf4j, target_dir=slf4j_root)
+        self._tokenizer = StanfordSegmenter(java_class=java_class, path_to_jar=path_to_jar,
+                                            path_to_slf4j=path_to_slf4j, path_to_dict=path_to_dict,
+                                            path_to_sihan_corpora_dict=path_to_sihan_corpora_dict,
+                                            path_to_model=path_to_model)
+
+    def __call__(self, sample):
+        """
+
+        Parameters
+        ----------
+        sample: str
+            The Chinese sentence to tokenize. Better not to input sentence in other languages
+            since this class is mainly used for Chinese Word Segmentation.
+
+        Returns
+        -------
+        ret : list of strs
+            List of tokens
+        """
+        return [tok for tok in self._tokenizer.segment(sample).strip().split()]
diff --git a/gluonnlp/data/utils.py b/gluonnlp/data/utils.py
--- a/gluonnlp/data/utils.py
+++ b/gluonnlp/data/utils.py
@@ -26,6 +26,7 @@
 import os
 import collections
 import zipfile
+import tarfile
 import numpy as np
 
 from mxnet.gluon.data import SimpleDataset
@@ -245,6 +246,7 @@ def short_hash(name):
         raise ValueError('Vocabulary for {name} is not available.'.format(name=name))
     return _vocab_sha1[name][:8]
 
+
 def _load_pretrained_vocab(name, root=os.path.join('~', '.mxnet', 'models')):
     """Load the accompanying vocabulary object for pretrained model.
 
@@ -292,6 +294,7 @@ def _load_pretrained_vocab(name, root=os.path.join('~', '.mxnet', 'models')):
     else:
         raise ValueError('Downloaded file has different hash. Please try again.')
 
+
 def _load_vocab_file(file_path):
     with open(file_path, 'r') as f:
         from ..vocab import Vocab
@@ -304,3 +307,24 @@ def _get_home_dir():
     # expand ~ to actual path
     _home_dir = os.path.expanduser(_home_dir)
     return _home_dir
+
+
+def _extract_archive(file, target_dir):
+    """Extract archive file
+
+    Parameters
+    ----------
+    file : str
+        Absolute path of the archive file.
+    target_dir : str
+        Target directory of the archive to be uncompressed
+
+    """
+    if file.endswith('.gz') or file.endswith('.tar') or file.endswith('.tgz'):
+        archive = tarfile.open(file, 'r')
+    elif file.endswith('.zip'):
+        archive = zipfile.ZipFile(file, 'r')
+    else:
+        raise Exception('Unrecognized file type: ' + file)
+    archive.extractall(path=target_dir)
+    archive.close()
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -57,6 +57,7 @@ def find_version(*file_paths):
             'nltk==3.2.5',
             'scipy',
             'numba',
+            'jieba',
         ],
         'dev': [
             'pytest',


Test patch: