| | """ |
| | A module for interfacing with ``tokenizer.perl`` from Moses. |
| | |
| | Copyright ® 2016-2017, Luís Gomes <luismsgomes@gmail.com> |
| | """ |
| |
|
| | usage = """ |
| | Usage: |
| | moses-tokenizer [options] <lang> [<inputfile> [<outputfile>]] |
| | moses-tokenizer --selftest [--verbose] |
| | |
| | Options: |
| | --selftest, -t Run selftests. |
| | --verbose, -v Be more verbose. |
| | --old Use older version (1.0) of the tokenizer. |
| | If this option is not given, then version 1.1 |
| | will be used. |
| | |
| | 2016, Luís Gomes <luismsgomes@gmail.com> |
| | """ |
| |
|
| |
|
| | from docopt import docopt |
| | from openfile import openfile |
| | from os import path |
| | from toolwrapper import ToolWrapper |
| | import sys |
| |
|
| |
|
| | class MosesTokenizer(ToolWrapper): |
| | """A module for interfacing with ``tokenizer.perl`` from Moses. |
| | |
| | This class communicates with tokenizer.perl process via pipes. When the |
| | MosesTokenizer object is no longer needed, the close() method should be |
| | called to free system resources. The class supports the context manager |
| | interface. If used in a with statement, the close() method is invoked |
| | automatically. |
| | |
| | >>> tokenize = MosesTokenizer('en') |
| | >>> tokenize('Hello World!') |
| | ['Hello', 'World', '!'] |
| | """ |
| |
|
| | def __init__(self, lang="en"): |
| | self.lang = lang |
| | program = path.join( |
| | path.dirname(__file__), |
| | "../tokenizer.perl" |
| | ) |
| | argv = ["perl", program, "-q", "-l", self.lang] |
| |
|
| | |
| | |
| | argv.extend(["-b", "-a"]) |
| | super().__init__(argv) |
| |
|
| | def __str__(self): |
| | return "MosesTokenizer(lang=\"{lang}\")".format(lang=self.lang) |
| |
|
| | def __call__(self, sentence): |
| | """Tokenizes a single sentence. |
| | |
| | Newline characters are not allowed in the sentence to be tokenized. |
| | """ |
| | assert isinstance(sentence, str) |
| | sentence = sentence.rstrip("\n") |
| | assert "\n" not in sentence |
| | if not sentence: |
| | return [] |
| | self.writeline(sentence) |
| | return self.readline().split() |
| |
|
| |
|
| | def main(): |
| | args = docopt(usage) |
| | if args["--selftest"]: |
| | import doctest |
| | import mosestokenizer.tokenizer |
| | doctest.testmod(mosestokenizer.tokenizer) |
| | if not args["<lang>"]: |
| | sys.exit(0) |
| | tokenize = MosesTokenizer( |
| | args["<lang>"] |
| | ) |
| | inputfile = openfile(args["<inputfile>"]) |
| | outputfile = openfile(args["<outputfile>"], "wt") |
| | with inputfile, outputfile: |
| | for line in inputfile: |
| | print(*tokenize(line), file=outputfile) |
| |
|
| | if __name__ == "__main__": |
| | main() |
| |
|
