Repository: WorksApplications/sudachi.rs
Pull request #7 (2020-08-22 05:28:56)

Add bake_dictionary option
Originally, the dictionary at `src/resources/system.dic` needed to exist to compile the crate.
With this change, you can get the same behavior by enabling the `bake_dictionary` feature, but it is not enabled by default.
This makes compiles faster and will 

I documented this in the English README. The Japanese README still needs some translations--my 日本語 skills are too basic :sweat_smile:.

Other changes:
- Code formatting
- `Mode` enum: add documentation and `Copy` trait derive

Please let me know your thoughts--I'm open to suggestions.
Thanks!


Patch:
diff --git a/.gitignore b/.gitignore
index aa0598d..f611cbe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /target
 **/*.rs.bk
 *.dic
+README*.html
diff --git a/Cargo.lock b/Cargo.lock
index 8480f81..67fa45e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -23,6 +23,11 @@ name = "bitflags"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "clap"
 version = "2.32.0"
@@ -45,6 +50,11 @@ dependencies = [
  "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "libc"
 version = "0.2.50"
@@ -72,6 +82,14 @@ dependencies = [
  "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "proc-macro2"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "quote"
 version = "0.6.11"
@@ -80,6 +98,14 @@ dependencies = [
  "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "quote"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.51"
@@ -122,8 +148,11 @@ dependencies = [
 name = "sudachi"
 version = "0.1.0"
 dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "nom 4.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -136,6 +165,16 @@ dependencies = [
  "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "syn"
+version = "1.0.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "termion"
 version = "1.5.1"
@@ -154,6 +193,24 @@ dependencies = [
  "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "thiserror-impl 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.2.1"
@@ -169,6 +226,11 @@ name = "unicode-xid"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "vec_map"
 version = "0.8.1"
@@ -202,24 +264,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
 "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
 "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
+"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 "checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e"
 "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
+"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 "checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1"
 "checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39"
 "checksum nom 4.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "22293d25d3f33a8567cc8a1dc20f40c7eeb761ce83d0fcca059858580790cac3"
 "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915"
+"checksum proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12"
 "checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1"
+"checksum quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
 "checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85"
 "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
 "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
 "checksum structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "3d0760c312538987d363c36c42339b55f5ee176ea8808bbe4543d484a291c8d1"
 "checksum structopt-derive 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "528aeb7351d042e6ffbc2a6fb76a86f9b622fdf7c25932798e7a82cb03bc94c6"
 "checksum syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1825685f977249735d510a242a6727b46efe914bb67e38d30c071b1b72b1d5c2"
+"checksum syn 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9"
 "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
 "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
+"checksum thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
+"checksum thiserror-impl 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
 "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1"
 "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
+"checksum unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
 "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
 "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
 "checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0"
diff --git a/Cargo.toml b/Cargo.toml
index 5a25642..85940fb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,5 +6,14 @@ edition = "2018"
 license = "Apache-2.0"
 
 [dependencies]
-structopt = "0.2"
+cfg-if = "0.1"
 nom = "4.2.2"
+structopt = "0.2"
+thiserror = "1.0"
+
+[dev-dependencies]
+lazy_static = "1.4.0"
+
+[features]
+# Compile dictionary into executable (specifying dictionary becomes optional)
+bake_dictionary = []
diff --git a/README.ja.md b/README.ja.md
new file mode 100644
index 0000000..dd28c7a
--- /dev/null
+++ b/README.ja.md
@@ -0,0 +1,190 @@
+# sudachi.rs - 日本語README
+
+<p align="center"><img width="100" src="logo.png" alt="sudachi.rs logo"></p>
+
+形態素解析器 [Sudachi](https://github.com/WorksApplications/Sudachi)  - 非公式 Rust 🦀 クローン
+
+[English README](README.md)
+
+## 注意
+
+これはRust勉強のための趣味実装で、実装が未完の部分があります。特に、未知語が存在するときにエラーが発生します（ラティスで最初から最後までパスが存在しない場合）。
+
+```sh
+$ echo "あ" | sudachi
+あ      感動詞,フィラー,*,*,*,* あー
+EOS
+
+$ echo "阿" | sudachi
+thread 'main' panicked at 'EOS isn't connected to BOS', src/lattice.rs:70:13
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+```
+
+他の方によるRust実装も参照ください; [Yasu-umi/sudachiclone-rs](https://github.com/Yasu-umi/sudachiclone-rs)
+
+
+## 利用例
+
+複数粒度での分割
+
+```
+$ echo 選挙管理委員会 | sudachi
+選挙管理委員会	名詞,固有名詞,一般,*,*,*	選挙管理委員会
+EOS
+
+$ echo 選挙管理委員会 | sudachi --mode A
+選挙	名詞,普通名詞,サ変可能,*,*,*	選挙
+管理	名詞,普通名詞,サ変可能,*,*,*	管理
+委員	名詞,普通名詞,一般,*,*,*	委員
+会	名詞,普通名詞,一般,*,*,*	会
+EOS
+```
+
+正規化表記
+
+```
+$ echo 打込む かつ丼 附属 vintage | sudachi
+打込む	動詞,一般,*,*,五段-マ行,終止形-一般	打ち込む
+ 	空白,*,*,*,*,*
+かつ丼	名詞,普通名詞,一般,*,*,*	カツ丼
+ 	空白,*,*,*,*,*
+附属	名詞,普通名詞,サ変可能,*,*,*	付属
+ 	空白,*,*,*,*,*
+vintage	名詞,普通名詞,一般,*,*,*	ビンテージ
+```
+
+分かち書き出力
+
+```
+$ cat lemon.txt
+えたいの知れない不吉な塊が私の心を始終圧えつけていた。
+焦躁と言おうか、嫌悪と言おうか――酒を飲んだあとに宿酔があるように、酒を毎日飲んでいると宿酔に相当した時期がやって来る。
+それが来たのだ。これはちょっといけなかった。
+
+$ sudachi --wakati lemon.txt
+えたい の 知れ ない 不吉 な 塊 が 私 の 心 を 始終 圧え つけ て い た 。
+焦躁 と 言おう か 、 嫌悪 と 言おう か ― ― 酒 を 飲ん だ あと に 宿酔 が ある よう に 、 酒 を 毎日 飲ん で いる と 宿酔 に 相当 し た 時期 が やっ て 来る 。
+それ が 来 た の だ 。 これ は ちょっと いけ なかっ た 。
+```
+
+## 利用方法
+
+```
+$ sudachi -h
+sudachi 0.1.0
+A Japanese tokenizer
+
+USAGE:
+    sudachi [FLAGS] [OPTIONS] --dict <dictionary_path> [file]
+
+FLAGS:
+    -d, --debug      Debug mode: Dumps lattice
+    -h, --help       Prints help information
+    -a, --all        Prints all fields
+    -V, --version    Prints version information
+    -w, --wakati     Outputs only surface form
+
+OPTIONS:
+    -l, --dict <dictionary_path>    Path to sudachi dictionary
+    -m, --mode <mode>               Split unit: "A" (short), "B" (middle), or "C" (Named Entity) [default: C]
+
+ARGS:
+    <file>    Input text file: If not present, read from STDIN
+```
+
+## セットアップ
+
+### 1. ソースコードの取得
+
+```
+$ git clone https://github.com/sorami/sudachi.rs.git
+```
+
+### 2. Sudachi辞書のダウンロード
+
+<!-- START: translate from english -->
+[WorksApplications/SudachiDict](https://github.com/WorksApplications/SudachiDict)から辞書のzipファイル（ `small` 、 `core` 、 `full` から一つ選択）し、解凍して、中にある `system_*.dic` ファイルを `src/resources/system.dic` として置いてください （ファイル名が `system.dic` に変わっていることに注意）。
+
+上記のように手動で設置する以外に、レポジトリにあるスクリプトを使って自動的に `core` 辞書をダウンロードし `src/resources/system.dic` として設置することもできます。
+
+#### Convenience Script
+
+Optionally, you can use the [`fetch_dictionary.sh`](fetch_dictionary.sh) shell script to download the `core` dictionary and install it to `src/resources/system.dic`.
+
+```
+$ ./fetch_dictionary.sh
+```
+
+### 3. ビルド
+
+#### Build (default)
+
+```
+$ cargo build --release
+```
+
+#### Build (bake dictionary into binary)
+
+Specify the `bake_dictionary` feature to avoid the requirement for the `--dict` argument with each invocation.
+ビルドされた実行ファイルは、**辞書バイナリを内包しています**。
+The `--dict` option will be optional (default to using the integrated dictionary).
+
+You must specify the path the dictionary file in the `SUDACHI_DICT_PATH` environment variable when building.
+`SUDACHI_DICT_PATH` is relative to the `src/` directory (or absolute).
+
+Example on Unix-like system:
+```sh
+# Download dictionary to src/resources/system.dic
+$ ./fetch_dictionary.sh
+
+# Build with bake_dictionary feature (relative to src/ path)
+$ env SUDACHI_DICT_PATH=resources/system.dic cargo build --release --features bake_dictionary
+
+# もしくは
+
+# Build with bake_dictionary feature (absolute path)
+$ env SUDACHI_DICT_PATH=/path/to/my-sudachi.dic cargo build --release --features bake_dictionary
+```
+
+<!-- END: translate from english -->
+
+
+### 4. インストール
+```
+sudachi.rs/ $ cargo install --path .
+
+$ which sudachi
+/Users/<USER>/.cargo/bin/sudachi
+
+$ sudachi -h
+sudachi 0.1.0
+A Japanese tokenizer
+...
+```
+
+
+## ToDo
+
+- [ ] 未知語処理
+- [ ] 簡単な辞書ファイルのインストール、管理（[SudachiPyでの方式を参考に](https://github.com/WorksApplications/SudachiPy/issues/73)）
+- [ ] crates.io への登録
+
+
+## リファレンス
+
+### Sudachi
+
+- [WorksApplications/Sudachi](https://github.com/WorksApplications/Sudachi)
+- [WorksApplications/SudachiDict](https://github.com/WorksApplications/SudachiDict)
+- [WorksApplications/SudachiPy](https://github.com/WorksApplications/SudachiPy)
+- [msnoigrs/gosudachi](https://github.com/msnoigrs/gosudachi)
+
+### Rustによる形態素解析器の実装
+
+- [agatan/yoin: A Japanese Morphological Analyzer written in pure Rust](https://github.com/agatan/yoin)
+- [wareya/notmecab-rs: notmecab-rs is a very basic mecab clone, designed only to do parsing, not training.](https://github.com/wareya/notmecab-rs)
+
+### ロゴ
+
+- [Sudachiのロゴ](https://github.com/WorksApplications/Sudachi/blob/develop/docs/Sudachi.png)
+- カニのイラスト: [Pixabay](https://pixabay.com/ja/vectors/%E5%8B%95%E7%89%A9-%E3%82%AB%E3%83%8B-%E7%94%B2%E6%AE%BB%E9%A1%9E-%E6%B5%B7-2029728/)
diff --git a/README.md b/README.md
index 8463e38..c4a8205 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,11 @@
+# sudachi.rs - English README
 **2021-07-07 UPDATE: The official Sudachi team will take over this project** (cf. [日本語形態素解析器 SudachiPy の 現状と今後について - Speaker Deck](https://speakerdeck.com/waptech/ri-ben-yu-xing-tai-su-jie-xi-qi-sudachipy-false-xian-zhuang-tojin-hou-nituite?slide=28))
 
-
-# sudachi.rs
-
 <p align="center"><img width="100" src="logo.png" alt="sudachi.rs logo"></p>
 
 An official [Sudachi](https://github.com/WorksApplications/Sudachi) clone in Rust 🦀
 
-[日本語 README](#sudachirs---日本語readme)
-
+[日本語 README](README.ja.md)
 
 ## Caution
 
@@ -79,7 +76,7 @@ sudachi 0.1.0
 A Japanese tokenizer
 
 USAGE:
-    sudachi [FLAGS] [OPTIONS] [file]
+    sudachi [FLAGS] [OPTIONS] --dict <dictionary_path> [file]
 
 FLAGS:
     -d, --debug      Debug mode: Dumps lattice
@@ -89,7 +86,8 @@ FLAGS:
     -w, --wakati     Outputs only surface form
 
 OPTIONS:
-    -m, --mode <mode>    Split unit: "A" (short), "B" (middle), or "C" (Named Entity) [default: C]
+    -l, --dict <dictionary_path>    Path to sudachi dictionary
+    -m, --mode <mode>               Split unit: "A" (short), "B" (middle), or "C" (Named Entity) [default: C]
 
 ARGS:
     <file>    Input text file: If not present, read from STDIN
@@ -105,188 +103,50 @@ $ git clone https://github.com/sorami/sudachi.rs.git
 
 ### 2. Download a Sudachi Dictionary
 
-You can download a dictionary zip file from [WorksApplications/SudachiDict](https://github.com/WorksApplications/SudachiDict) (choose one from `small`, `core`, or `full`), unzip it, and place the `system_*.dic` file to `src/resources/system.dic` (Note that the file name is changed to `system.dic`) .
+Sudachi requires a dictionary to operate.
+You can download a dictionary ZIP file from [WorksApplications/SudachiDict](https://github.com/WorksApplications/SudachiDict) (choose one from `small`, `core`, or `full`), unzip it, and place the `system_*.dic` file somewhere.
 
-Alternatively, you can use a quick shell script in the source code; This script will download the `core` dictionary and place it to `src/resources/system.dic`.
+#### Convenience Script
+
+Optionally, you can use the [`fetch_dictionary.sh`](fetch_dictionary.sh) shell script to download the `core` dictionary and install it to `src/resources/system.dic`.
 
 ```
 $ ./fetch_dictionary.sh
 ```
 
-### 3. Build, Install
+### 3. Build
 
-The built executable will **contain the dictionary binary**.
+#### Build (default)
 
 ```
-$ cargo build
+$ cargo build --release
 ```
 
-or
-
-```
-sudachi.rs/ $ cargo install --path .
+#### Build (bake dictionary into binary)
 
-$ which sudachi
-/Users/<USER>/.cargo/bin/sudachi
+Specify the `bake_dictionary` feature to avoid the requirement for the `--dict` argument with each invocation.
+The `sudachi` executable will **contain the dictionary binary**.
+The `--dict` option will be optional (default to using the integrated dictionary).
 
-$ sudachi -h
-sudachi 0.1.0
-A Japanese tokenizer
-...
-```
-
-
-## ToDo
-
-- [ ] Out of Vocabulary handling
-- [ ] Easy dictionary file install & management, [similar to SudachiPy](https://github.com/WorksApplications/SudachiPy/issues/73)
-- [ ] Registration to crates.io
-
-
-## References
-
-### Sudachi
-
-- [WorksApplications/Sudachi](https://github.com/WorksApplications/Sudachi)
-- [WorksApplications/SudachiDict](https://github.com/WorksApplications/SudachiDict)
-- [WorksApplications/SudachiPy](https://github.com/WorksApplications/SudachiPy)
-- [msnoigrs/gosudachi](https://github.com/msnoigrs/gosudachi)
-
-
-### Morphological Analyzers in Rust
-
-- [agatan/yoin: A Japanese Morphological Analyzer written in pure Rust](https://github.com/agatan/yoin)
-- [wareya/notmecab-rs: notmecab-rs is a very basic mecab clone, designed only to do parsing, not training.](https://github.com/wareya/notmecab-rs)
-
-### Logo
-
-- [Sudachi Logo](https://github.com/WorksApplications/Sudachi/blob/develop/docs/Sudachi.png)
-- Crab illustration: [Pixabay](https://pixabay.com/ja/vectors/%E5%8B%95%E7%89%A9-%E3%82%AB%E3%83%8B-%E7%94%B2%E6%AE%BB%E9%A1%9E-%E6%B5%B7-2029728/)
-
-
-***
-
-
-# sudachi.rs - 日本語README
-
-<p align="center"><img width="100" src="logo.png" alt="sudachi.rs logo"></p>
-
-形態素解析器 [Sudachi](https://github.com/WorksApplications/Sudachi)  - 公式 Rust 🦀 クローン
-
-[English README](#sudachirs)
-
-## 注意
-
-これはRust勉強のための趣味実装で、実装が未完の部分があります。特に、未知語が存在するときにエラーが発生します（ラティスで最初から最後までパスが存在しない場合）。
+You must specify the path the dictionary file in the `SUDACHI_DICT_PATH` environment variable when building.
+`SUDACHI_DICT_PATH` is relative to the `src/` directory (or absolute).
 
+Example on Unix-like system:
 ```sh
-$ echo "あ" | sudachi
-あ      感動詞,フィラー,*,*,*,* あー
-EOS
-
-$ echo "阿" | sudachi
-thread 'main' panicked at 'EOS isn't connected to BOS', src/lattice.rs:70:13
-note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
-```
-
-他の方によるRust実装も参照ください; [Yasu-umi/sudachiclone-rs](https://github.com/Yasu-umi/sudachiclone-rs)
-
-
-## 利用例
-
-複数粒度での分割
-
-```
-$ echo 選挙管理委員会 | sudachi
-選挙管理委員会	名詞,固有名詞,一般,*,*,*	選挙管理委員会
-EOS
-
-$ echo 選挙管理委員会 | sudachi --mode A
-選挙	名詞,普通名詞,サ変可能,*,*,*	選挙
-管理	名詞,普通名詞,サ変可能,*,*,*	管理
-委員	名詞,普通名詞,一般,*,*,*	委員
-会	名詞,普通名詞,一般,*,*,*	会
-EOS
-```
-
-正規化表記
-
-```
-$ echo 打込む かつ丼 附属 vintage | sudachi
-打込む	動詞,一般,*,*,五段-マ行,終止形-一般	打ち込む
- 	空白,*,*,*,*,*
-かつ丼	名詞,普通名詞,一般,*,*,*	カツ丼
- 	空白,*,*,*,*,*
-附属	名詞,普通名詞,サ変可能,*,*,*	付属
- 	空白,*,*,*,*,*
-vintage	名詞,普通名詞,一般,*,*,*	ビンテージ
-```
-
-分かち書き出力
-
-```
-$ cat lemon.txt
-えたいの知れない不吉な塊が私の心を始終圧えつけていた。
-焦躁と言おうか、嫌悪と言おうか――酒を飲んだあとに宿酔があるように、酒を毎日飲んでいると宿酔に相当した時期がやって来る。
-それが来たのだ。これはちょっといけなかった。
-
-$ sudachi --wakati lemon.txt
-えたい の 知れ ない 不吉 な 塊 が 私 の 心 を 始終 圧え つけ て い た 。
-焦躁 と 言おう か 、 嫌悪 と 言おう か ― ― 酒 を 飲ん だ あと に 宿酔 が ある よう に 、 酒 を 毎日 飲ん で いる と 宿酔 に 相当 し た 時期 が やっ て 来る 。
-それ が 来 た の だ 。 これ は ちょっと いけ なかっ た 。
-```
-
-## 利用方法
-
-```
-$ sudachi -h
-sudachi 0.1.0
-A Japanese tokenizer
-
-USAGE:
-    sudachi [FLAGS] [OPTIONS] [file]
-
-FLAGS:
-    -d, --debug      Debug mode: Dumps lattice
-    -h, --help       Prints help information
-    -a, --all        Prints all fields
-    -V, --version    Prints version information
-    -w, --wakati     Outputs only surface form
-
-OPTIONS:
-    -m, --mode <mode>    Split unit: "A" (short), "B" (middle), or "C" (Named Entity) [default: C]
-
-ARGS:
-    <file>    Input text file: If not present, read from STDIN
-```
-
-## セットアップ
-
-### 1. ソースコードの取得
-
-```
-$ git clone https://github.com/sorami/sudachi.rs.git
-```
-
-### 2. Sudachi辞書のダウンロード 
-
-[WorksApplications/SudachiDict](https://github.com/WorksApplications/SudachiDict)から辞書のzipファイル（ `small` 、 `core` 、 `full` から一つ選択）し、解凍して、中にある `system_*.dic` ファイルを `src/resources/system.dic` として置いてください （ファイル名が `system.dic` に変わっていることに注意）。
-
-上記のように手動で設置する以外に、レポジトリにあるスクリプトを使って自動的に `core` 辞書をダウンロードし `src/resources/system.dic` として設置することもできます。
-
-```
+# Download dictionary to src/resources/system.dic
 $ ./fetch_dictionary.sh
-```
 
-### 3. ビルド、インストール
+# Build with bake_dictionary feature (relative to src/ path)
+$ env SUDACHI_DICT_PATH=resources/system.dic cargo build --release --features bake_dictionary
 
-ビルドされた実行ファイルは、**辞書バイナリを内包しています**。
+# or
 
+# Build with bake_dictionary feature (absolute path)
+$ env SUDACHI_DICT_PATH=/path/to/my-sudachi.dic cargo build --release --features bake_dictionary
 ```
-$ cargo build
-```
 
-もしくは
+
+### 4. Install
 
 ```
 sudachi.rs/ $ cargo install --path .
@@ -303,12 +163,12 @@ A Japanese tokenizer
 
 ## ToDo
 
-- [ ] 未知語処理
-- [ ] 簡単な辞書ファイルのインストール、管理（[SudachiPyでの方式を参考に](https://github.com/WorksApplications/SudachiPy/issues/73)）
-- [ ] crates.io への登録
+- [ ] Out of Vocabulary handling
+- [ ] Easy dictionary file install & management, [similar to SudachiPy](https://github.com/WorksApplications/SudachiPy/issues/73)
+- [ ] Registration to crates.io
 
 
-## リファレンス
+## References
 
 ### Sudachi
 
@@ -317,12 +177,13 @@ A Japanese tokenizer
 - [WorksApplications/SudachiPy](https://github.com/WorksApplications/SudachiPy)
 - [msnoigrs/gosudachi](https://github.com/msnoigrs/gosudachi)
 
-### Rustによる形態素解析器の実装
+
+### Morphological Analyzers in Rust
 
 - [agatan/yoin: A Japanese Morphological Analyzer written in pure Rust](https://github.com/agatan/yoin)
 - [wareya/notmecab-rs: notmecab-rs is a very basic mecab clone, designed only to do parsing, not training.](https://github.com/wareya/notmecab-rs)
 
-### ロゴ
+### Logo
 
-- [Sudachiのロゴ](https://github.com/WorksApplications/Sudachi/blob/develop/docs/Sudachi.png)
-- カニのイラスト: [Pixabay](https://pixabay.com/ja/vectors/%E5%8B%95%E7%89%A9-%E3%82%AB%E3%83%8B-%E7%94%B2%E6%AE%BB%E9%A1%9E-%E6%B5%B7-2029728/)
+- [Sudachi Logo](https://github.com/WorksApplications/Sudachi/blob/develop/docs/Sudachi.png)
+- Crab illustration: [Pixabay](https://pixabay.com/ja/vectors/%E5%8B%95%E7%89%A9-%E3%82%AB%E3%83%8B-%E7%94%B2%E6%AE%BB%E9%A1%9E-%E6%B5%B7-2029728/)
diff --git a/src/dic.rs b/src/dic.rs
index 88016ca..69529f9 100644
--- a/src/dic.rs
+++ b/src/dic.rs
@@ -1,3 +1,20 @@
+use nom::{le_u16, le_u8};
+
+use crate::error::SudachiNomCustomError;
+
 pub mod grammar;
 pub mod header;
 pub mod lexicon;
+
+named!(
+    utf16_string<&[u8], String>,
+    do_parse!(
+        length: le_u8 >>
+        v: count!(le_u16, length as usize) >>
+
+        (String::from_utf16(&v)
+            .map_err(|_| nom::Err::Failure(
+                nom::Context::Code(&[] as &[u8], nom::ErrorKind::Custom(SudachiNomCustomError::FromUtf16Nom as u32))))?
+        )
+    )
+);
diff --git a/src/dic/grammar.rs b/src/dic/grammar.rs
index 74f8bb1..cb6460c 100644
--- a/src/dic/grammar.rs
+++ b/src/dic/grammar.rs
@@ -1,11 +1,14 @@
-use nom::{le_i16, le_u16, le_u8};
+use nom::{le_i16, le_u16};
+
+use crate::dic::utf16_string;
+use crate::prelude::*;
 
 pub struct Grammar<'a> {
     bytes: &'a [u8],
     pub pos_list: Vec<Vec<String>>,
     connect_table_offset: usize,
     left_id_size: i16,
-    right_id_size: i16,
+    _right_id_size: i16,
 
     pub storage_size: usize,
 }
@@ -16,47 +19,37 @@ impl<'a> Grammar<'a> {
     pub const BOS_PARAMETER: (i16, i16, i16) = (0, 0, 0); // left_id, right_id, cost
     pub const EOS_PARAMETER: (i16, i16, i16) = (0, 0, 0); // left_id, right_id, cost
 
-    pub fn new(buf: &[u8], offset: usize) -> Grammar {
-        let (rest, (pos_list, left_id_size, right_id_size)) = grammar_parser(buf, offset).unwrap();
+    pub fn new(buf: &[u8], offset: usize) -> SudachiResult<Grammar> {
+        let (rest, (pos_list, left_id_size, right_id_size)) =
+            grammar_parser(buf, offset).map_err(|_| SudachiError::InvalidDictionaryGrammar)?;
 
         let connect_table_offset = buf.len() - rest.len();
         let storage_size =
             (connect_table_offset - offset) + 2 * left_id_size as usize * right_id_size as usize;
 
-        Grammar {
+        Ok(Grammar {
             bytes: buf,
             pos_list,
             connect_table_offset,
             left_id_size,
-            right_id_size,
+            _right_id_size: right_id_size,
             storage_size,
-        }
+        })
     }
 
-    pub fn get_connect_cost(&self, left_id: i16, right_id: i16) -> i16 {
+    pub fn get_connect_cost(&self, left_id: i16, right_id: i16) -> SudachiResult<i16> {
         let (_rest, connect_cost) = connect_cost_parser(
             self.bytes,
             self.connect_table_offset,
             left_id as usize,
             self.left_id_size as usize,
             right_id as usize,
-        )
-        .unwrap();
+        )?;
 
-        connect_cost
+        Ok(connect_cost)
     }
 }
 
-named!(
-    utf16_string<&[u8], String>,
-    do_parse!(
-        length: le_u8 >>
-        v: count!(le_u16, length as usize) >>
-
-        (String::from_utf16(&v).unwrap())
-    )
-);
-
 named_args!(
     grammar_parser(offset: usize)<&[u8], (Vec<Vec<String>>, i16, i16)>,
     do_parse!(
diff --git a/src/dic/header.rs b/src/dic/header.rs
index e516c2d..185b966 100644
--- a/src/dic/header.rs
+++ b/src/dic/header.rs
@@ -1,36 +1,105 @@
-use std::str;
-
 use nom::le_u64;
+use thiserror::Error;
+
+/// Sudachi error
+#[derive(Error, Debug, Eq, PartialEq)]
+#[non_exhaustive]
+pub enum HeaderError {
+    #[error("Invalid header")]
+    InvalidVersion,
 
+    #[error("Unable to parse")]
+    CannotParse,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub struct Header {
     pub version: u64,
-    create_time: u64,
+    _create_time: u64,
     description: String,
 }
 
 impl Header {
     const DESCRIPTION_SIZE: usize = 256;
+    const EXPECTED_VERSION: u64 = 0x7366d3f18bd111e7;
     pub const STORAGE_SIZE: usize = 8 + 8 + Header::DESCRIPTION_SIZE;
 
-    pub fn new(bytes: &[u8], offset: usize) -> Header {
-        let (_rest, header) = header_parser(bytes, offset).unwrap();
-        assert_eq!(header.version, 0x7366d3f18bd111e7);
+    pub fn new(bytes: &[u8]) -> Result<(&[u8], Header), HeaderError> {
+        let (rest, header) = header_parser(bytes).map_err(|_| HeaderError::CannotParse)?;
+
+        if header.version != Self::EXPECTED_VERSION {
+            return Err(HeaderError::InvalidVersion);
+        }
 
-        header
+        Ok((rest, header))
     }
 }
 
+/// Create String from UTF-8 bytes up to NUL byte or end of slice (whichever is first)
+fn nul_terminated_str_from_slice(buf: &[u8]) -> String {
+    let str_bytes: &[u8] = if let Some(nul_idx) = buf.iter().position(|b| *b == 0) {
+        &buf[..nul_idx]
+    } else {
+        &buf
+    };
+    String::from_utf8_lossy(str_bytes).to_string()
+}
+
 named_args!(
-    header_parser(offset: usize)<&[u8], Header>,
+    header_parser()<&[u8], Header>,
     do_parse!(
-        _seek: take!(offset) >>
         version: le_u64 >>
         create_time: le_u64 >>
         desc_buf: take!(Header::DESCRIPTION_SIZE) >>
 
         (Header{ version,
-                 create_time,
-                 description: str::from_utf8(&desc_buf).unwrap().to_string() })
-                 // alternative: lossy_utf8, unchecked
+                 _create_time: create_time,
+                 description: nul_terminated_str_from_slice(&desc_buf) })
     )
 );
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn header_from_parts<T: AsRef<[u8]>>(
+        version: u64,
+        create_time: u64,
+        description: T,
+    ) -> Result<Header, HeaderError> {
+        let mut bytes = Vec::new();
+        bytes.extend(&version.to_le_bytes());
+        bytes.extend(&create_time.to_le_bytes());
+        bytes.extend(description.as_ref());
+
+        Header::new(&bytes).map(|(_rest, header)| header)
+    }
+
+    #[test]
+    fn graceful_failure() {
+        // Too small
+        assert_eq!(Header::new(&[]), Err(HeaderError::CannotParse));
+
+        assert_eq!(
+            header_from_parts(42, 0, vec![0; Header::DESCRIPTION_SIZE]),
+            Err(HeaderError::InvalidVersion)
+        );
+    }
+
+    #[test]
+    fn simple_header() {
+        let mut description: Vec<u8> = Vec::new();
+        let description_str = "My Description";
+        description.extend(description_str.bytes());
+        description.extend(&vec![0; Header::DESCRIPTION_SIZE]);
+
+        assert_eq!(
+            header_from_parts(Header::EXPECTED_VERSION, 1337, &description),
+            Ok(Header {
+                version: Header::EXPECTED_VERSION,
+                description: description_str.to_string(),
+                _create_time: 1337,
+            })
+        );
+    }
+}
diff --git a/src/dic/lexicon.rs b/src/dic/lexicon.rs
index 3173f46..a38dc93 100644
--- a/src/dic/lexicon.rs
+++ b/src/dic/lexicon.rs
@@ -9,6 +9,7 @@ use self::trie::Trie;
 use self::word_id_table::WordIdTable;
 use self::word_infos::{WordInfo, WordInfos};
 use self::word_params::WordParams;
+use crate::prelude::*;
 
 pub struct Lexicon<'a> {
     trie: Trie,
@@ -18,57 +19,57 @@ pub struct Lexicon<'a> {
 }
 
 impl<'a> Lexicon<'a> {
-    pub fn new(buf: &[u8], original_offset: usize) -> Lexicon {
+    pub fn new(buf: &[u8], original_offset: usize) -> SudachiResult<Lexicon> {
         let mut offset = original_offset;
 
-        let (_rest, trie_size) = parse_size(buf, offset).unwrap();
+        let (_rest, trie_size) = parse_size(buf, offset)?;
         offset += 4;
-        let (_rest, trie_array) = parse_trie_array(buf, offset, trie_size).unwrap();
+        let (_rest, trie_array) = parse_trie_array(buf, offset, trie_size)?;
         let trie = Trie::new(trie_array, trie_size);
         offset += trie.total_size();
 
-        let (_rest, word_id_table_size) = parse_size(buf, offset).unwrap();
+        let (_rest, word_id_table_size) = parse_size(buf, offset)?;
         let word_id_table = WordIdTable::new(buf, word_id_table_size, offset + 4);
         offset += word_id_table.storage_size();
 
-        let (_rest, word_params_size) = parse_size(buf, offset).unwrap();
+        let (_rest, word_params_size) = parse_size(buf, offset)?;
         let word_params = WordParams::new(buf, word_params_size, offset + 4);
         offset += word_params.storage_size();
 
         let word_infos = WordInfos::new(buf, offset, word_params.size());
 
-        Lexicon {
+        Ok(Lexicon {
             trie,
             word_id_table,
             word_params,
             word_infos,
-        }
+        })
     }
 
-    pub fn lookup(&self, input: &[u8], offset: usize) -> Vec<(u32, usize)> {
-        let result = self.trie.common_prefix_search(input, offset);
+    pub fn lookup(&self, input: &[u8], offset: usize) -> SudachiResult<Vec<(u32, usize)>> {
+        let result = self.trie.common_prefix_search(input, offset)?;
 
         let mut l: Vec<(u32, usize)> = Vec::new(); // (word_id, length)
         for item in result {
             let length = item.1;
-            for word_id in self.word_id_table.get(item.0) {
+            for word_id in self.word_id_table.get(item.0)? {
                 l.push((word_id, length));
             }
         }
 
-        l
+        Ok(l)
     }
 
-    pub fn get_word_info(&self, word_id: usize) -> WordInfo {
+    pub fn get_word_info(&self, word_id: usize) -> SudachiResult<WordInfo> {
         self.word_infos.get_word_info(word_id)
     }
 
-    pub fn get_word_param(&self, word_id: usize) -> (i16, i16, i16) {
-        let left_id = self.word_params.get_left_id(word_id);
-        let right_id = self.word_params.get_right_id(word_id);
-        let cost = self.word_params.get_cost(word_id);
+    pub fn get_word_param(&self, word_id: usize) -> SudachiResult<(i16, i16, i16)> {
+        let left_id = self.word_params.get_left_id(word_id)?;
+        let right_id = self.word_params.get_right_id(word_id)?;
+        let cost = self.word_params.get_cost(word_id)?;
 
-        (left_id, right_id, cost)
+        Ok((left_id, right_id, cost))
     }
 }
 
diff --git a/src/dic/lexicon/trie.rs b/src/dic/lexicon/trie.rs
index 38d3cfb..38071de 100644
--- a/src/dic/lexicon/trie.rs
+++ b/src/dic/lexicon/trie.rs
@@ -1,3 +1,5 @@
+use crate::prelude::*;
+
 pub struct Trie {
     array: Vec<u32>,
     size: u32, // number of elements
@@ -12,32 +14,48 @@ impl Trie {
         4 * self.size as usize
     }
 
-    pub fn common_prefix_search(&self, input: &[u8], offset: usize) -> Vec<(usize, usize)> {
+    pub fn common_prefix_search(
+        &self,
+        input: &[u8],
+        offset: usize,
+    ) -> SudachiResult<Vec<(usize, usize)>> {
         let mut result = Vec::new();
 
         let mut node_pos: usize = 0;
-        let mut unit: usize = *self.array.get(node_pos).unwrap() as usize;
+        let mut unit: usize = *self
+            .array
+            .get(node_pos)
+            .ok_or(SudachiError::MissingDictionaryTrie)? as usize;
         node_pos ^= Trie::offset(unit);
 
         for i in offset..input.len() {
-            let k = input.get(i).unwrap();
+            let k = input.get(i).ok_or(SudachiError::MissingDictionaryTrie)?;
             node_pos ^= *k as usize;
-            unit = *self.array.get(node_pos).unwrap() as usize;
+            unit = *self
+                .array
+                .get(node_pos)
+                .ok_or(SudachiError::MissingDictionaryTrie)? as usize;
             if Trie::label(unit) != *k as usize {
-                return result;
+                return Ok(result);
             }
 
             node_pos ^= Trie::offset(unit);
             if Trie::has_leaf(unit) {
                 let r = (
-                    Trie::value(*self.array.get(node_pos).unwrap() as usize),
+                    Trie::value(
+                        *self
+                            .array
+                            .get(node_pos)
+                            .ok_or(SudachiError::MissingDictionaryTrie)?
+                            as usize,
+                    ),
                     i + 1,
                 );
                 result.push(r);
             }
         }
 
-        result
+        Ok(result)
     }
 
     fn has_leaf(unit: usize) -> bool {
diff --git a/src/dic/lexicon/word_id_table.rs b/src/dic/lexicon/word_id_table.rs
index dffe8fc..6734df5 100644
--- a/src/dic/lexicon/word_id_table.rs
+++ b/src/dic/lexicon/word_id_table.rs
@@ -1,5 +1,7 @@
 use nom::{le_u32, le_u8};
 
+use crate::prelude::*;
+
 pub struct WordIdTable<'a> {
     bytes: &'a [u8],
     size: u32,
@@ -19,9 +21,9 @@ impl<'a> WordIdTable<'a> {
         4 + self.size as usize
     }
 
-    pub fn get(&self, index: usize) -> Vec<u32> {
-        let (_rest, result) = word_id_table_parser(self.bytes, self.offset, index).unwrap();
-        result
+    pub fn get(&self, index: usize) -> SudachiResult<Vec<u32>> {
+        let (_rest, result) = word_id_table_parser(self.bytes, self.offset, index)?;
+        Ok(result)
     }
 }
 
diff --git a/src/dic/lexicon/word_infos.rs b/src/dic/lexicon/word_infos.rs
index d32baff..323171f 100644
--- a/src/dic/lexicon/word_infos.rs
+++ b/src/dic/lexicon/word_infos.rs
@@ -1,48 +1,39 @@
 use nom::{le_i32, le_u16, le_u32, le_u8};
 
+use crate::dic::utf16_string;
+use crate::prelude::*;
+
 pub struct WordInfos<'a> {
     bytes: &'a [u8],
     offset: usize,
-    word_size: u32,
+    _word_size: u32,
 }
 
 impl<'a> WordInfos<'a> {
-    pub fn new(bytes: &'a [u8], offset: usize, word_size: u32) -> WordInfos {
+    pub fn new(bytes: &'a [u8], offset: usize, _word_size: u32) -> WordInfos {
         WordInfos {
             bytes,
             offset,
-            word_size,
+            _word_size,
         }
     }
 
-    pub fn get_word_info(&self, word_id: usize) -> WordInfo {
-        let index = le_u32(&self.bytes[self.offset + (4 * word_id)..])
-            .unwrap()
-            .1 as usize; // wordIdToOffset()
-        let mut word_info = word_info_parser(self.bytes, index).unwrap().1;
+    pub fn get_word_info(&self, word_id: usize) -> SudachiResult<WordInfo> {
+        let index = le_u32(&self.bytes[self.offset + (4 * word_id)..])?.1 as usize; // wordIdToOffset()
+        let mut word_info = word_info_parser(self.bytes, index)?.1;
 
         // TODO: can we set dictionary_form within the word_info_parser?
         let dfwi = word_info.dictionary_form_word_id;
         if (dfwi >= 0) & (dfwi != word_id as i32) {
-            word_info.dictionary_form = self.get_word_info(dfwi as usize).surface;
+            word_info.dictionary_form = self.get_word_info(dfwi as usize)?.surface;
         };
 
-        word_info
+        Ok(word_info)
     }
 
     // TODO: is_valid_split()
 }
 
-named!(
-    utf16_string<&[u8], String>,
-    do_parse!(
-        length: le_u8 >>
-        v: count!(le_u16, length as usize) >>
-
-        (String::from_utf16(&v).unwrap())
-    )
-);
-
 named_args!(
     word_info_parser(index: usize)<&[u8], WordInfo>,
     do_parse!(
diff --git a/src/dic/lexicon/word_params.rs b/src/dic/lexicon/word_params.rs
index e9e9c15..21d926e 100644
--- a/src/dic/lexicon/word_params.rs
+++ b/src/dic/lexicon/word_params.rs
@@ -1,5 +1,7 @@
 use nom::le_i16;
 
+use crate::prelude::*;
+
 pub struct WordParams<'a> {
     bytes: &'a [u8],
     size: u32,
@@ -25,28 +27,26 @@ impl<'a> WordParams<'a> {
         self.size
     }
 
-    pub fn get_left_id(&self, word_id: usize) -> i16 {
+    pub fn get_left_id(&self, word_id: usize) -> SudachiResult<i16> {
         let (_rest, num) =
-            i16_parser(self.bytes, self.offset + WordParams::ELEMENT_SIZE * word_id).unwrap();
-        num
+            i16_parser(self.bytes, self.offset + WordParams::ELEMENT_SIZE * word_id)?;
+        Ok(num)
     }
 
-    pub fn get_right_id(&self, word_id: usize) -> i16 {
+    pub fn get_right_id(&self, word_id: usize) -> SudachiResult<i16> {
         let (_rest, num) = i16_parser(
             self.bytes,
             self.offset + WordParams::ELEMENT_SIZE * word_id + 2,
-        )
-        .unwrap();
-        num
+        )?;
+        Ok(num)
     }
 
-    pub fn get_cost(&self, word_id: usize) -> i16 {
+    pub fn get_cost(&self, word_id: usize) -> SudachiResult<i16> {
         let (_rest, num) = i16_parser(
             self.bytes,
             self.offset + WordParams::ELEMENT_SIZE * word_id + 4,
-        )
-        .unwrap();
-        num
+        )?;
+        Ok(num)
     }
 }
 
diff --git a/src/error.rs b/src/error.rs
new file mode 100644
index 0000000..41240c2
--- /dev/null
+++ b/src/error.rs
@@ -0,0 +1,86 @@
+use std::convert::TryFrom;
+use std::fmt::Debug;
+
+use thiserror::Error;
+
+use crate::dic::header::HeaderError;
+
+pub type SudachiResult<T> = Result<T, SudachiError>;
+
+/// Sudachi error
+#[derive(Error, Debug)]
+#[non_exhaustive]
+pub enum SudachiError {
+    #[error("IO Error: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("Invalid header: {0}")]
+    InvalidHeader(#[from] HeaderError),
+
+    #[error("Invalid header")]
+    InvalidDictionaryGrammar,
+
+    #[error("Error parsing nom: {}", .0.description())]
+    NomParse(nom::ErrorKind<u32>),
+
+    #[error("Missing word_id")]
+    MissingWordId,
+
+    #[error("Missing part of speech")]
+    MissingPartOfSpeech,
+
+    #[error("Missing latice path")]
+    MissingLaticePath,
+
+    #[error("Missing dictionary trie")]
+    MissingDictionaryTrie,
+
+    #[error("Invalid UTF-16: {0}")]
+    FromUtf16(#[from] std::string::FromUtf16Error),
+
+    #[error("Invalid UTF-16 from nom")]
+    FromUtf16Nom,
+
+    #[error("End of sentence (EOS) is not connected to beginning of sentence (BOS)")]
+    EosBosDisconnect,
+}
+
+/// Define `SudachiNomCustomError` error and conversion to `SudachiError`.
+/// The variants that can be converted will have the same name.
+macro_rules! define_nom_error_enum {
+    (
+        $( $name:ident = $value_int:literal , )*
+    ) => {
+        #[derive(Debug, Copy, Clone, Eq, PartialEq)]
+        pub(crate) enum SudachiNomCustomError {
+            $($name = $value_int , )*
+        }
+
+        impl TryFrom<u32> for SudachiError {
+            type Error = ();
+            fn try_from(x: u32) -> Result<Self, Self::Error> {
+                match x {
+                    $( $value_int => Ok(SudachiError::$name) , )*
+                    _ => Err(()),
+                }
+            }
+        }
+    };
+}
+
+define_nom_error_enum! {
+    FromUtf16Nom = 0,
+}
+
+impl<I> From<nom::Err<I, u32>> for SudachiError {
+    fn from(err: nom::Err<I, u32>) -> Self {
+        if let nom::Err::Failure(nom::Context::Code(_v, nom::ErrorKind::Custom(custom_error))) =
+            &err
+        {
+            if let Ok(sudachi_error) = SudachiError::try_from(*custom_error) {
+                return sudachi_error;
+            }
+        }
+        SudachiError::NomParse(err.into_error_kind())
+    }
+}
diff --git a/src/lattice.rs b/src/lattice.rs
index 3e1ace2..9954a83 100644
--- a/src/lattice.rs
+++ b/src/lattice.rs
@@ -4,6 +4,7 @@ use std::i32;
 
 use self::node::Node;
 use crate::dic::grammar::Grammar;
+use crate::prelude::*;
 
 pub struct Lattice<'a> {
     grammar: &'a Grammar<'a>,
@@ -25,7 +26,7 @@ impl<'a> Lattice<'a> {
         }
     }
 
-    fn connect_node(&self, r_node: &mut Node) {
+    fn connect_node(&self, r_node: &mut Node) -> SudachiResult<()> {
         let begin = r_node.begin;
         r_node.total_cost = i32::MAX;
 
@@ -36,7 +37,7 @@ impl<'a> Lattice<'a> {
 
             let connect_cost = self
                 .grammar
-                .get_connect_cost(l_node.right_id, r_node.left_id);
+                .get_connect_cost(l_node.right_id, r_node.left_id)?;
             let cost = l_node.total_cost + connect_cost as i32;
             if cost < r_node.total_cost {
                 r_node.total_cost = cost;
@@ -45,43 +46,50 @@ impl<'a> Lattice<'a> {
         }
         r_node.total_cost += r_node.cost as i32;
 
-        r_node.is_connected_to_bos = match r_node.best_previous_node_index {
-            Some(_) => true,
-            None => false,
-        };
+        r_node.is_connected_to_bos = r_node.best_previous_node_index.is_some();
+
+        Ok(())
     }
 
-    pub fn insert(&mut self, begin: usize, end: usize, mut node: Node) {
+    pub fn insert(&mut self, begin: usize, end: usize, mut node: Node) -> SudachiResult<()> {
         node.set_range(begin, end);
-        self.connect_node(&mut node);
+        self.connect_node(&mut node)?;
         self.end_lists[end].push(node);
+
+        Ok(())
     }
 
-    pub fn connect_eos_node(&mut self) {
+    pub fn connect_eos_node(&mut self) -> SudachiResult<()> {
         let eos_node = Node::new_eos(self.size);
-        self.insert(eos_node.begin, eos_node.end, eos_node);
+        self.insert(eos_node.begin, eos_node.end, eos_node)
     }
 
-    pub fn get_best_path(&self) -> Vec<&Node> {
+    pub fn get_best_path(&self) -> SudachiResult<Vec<&Node>> {
         // TODO: reference of eos_node in struct `Lattice` ?
-        let eos_node = self.end_lists[self.size].last().unwrap();
+        let eos_node = self.end_lists[self.size]
+            .last()
+            .ok_or(SudachiError::MissingLaticePath)?;
 
         if !eos_node.is_connected_to_bos {
-            panic!("EOS isn't connected to BOS");
+            return Err(SudachiError::EosBosDisconnect);
         }
 
         let mut path = Vec::new();
 
         let mut node = eos_node;
         // todo
-        let (i, j) = node.best_previous_node_index.unwrap();
+        let (i, j) = node
+            .best_previous_node_index
+            .ok_or(SudachiError::MissingLaticePath)?;
         let mut i = i;
         let mut j = j;
         while (i, j) != (0, 0) {
             path.push(node);
 
             // todo
-            let (a, b) = node.best_previous_node_index.unwrap();
+            let (a, b) = node
+                .best_previous_node_index
+                .ok_or(SudachiError::MissingLaticePath)?;
             i = a;
             j = b;
             node = &self.end_lists[i][j];
@@ -90,6 +98,6 @@ impl<'a> Lattice<'a> {
         path.reverse();
         path.pop(); // EOS
 
-        path
+        Ok(path)
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 6b58e8f..4d36535 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,8 +1,32 @@
+//! Clone of [Sudachi](https://github.com/WorksApplications/Sudachi),
+//! a Japanese morphological analyzer
+//!
+//! The main entry point of the library is the
+//! [`Tokenizer`](tokenizer/struct.Tokenizer.html) struct, which
+//! implements [`Tokenize`](tokenizer/trait.Tokenize.html).
+
+#[cfg(test)]
+#[macro_use]
+extern crate lazy_static;
+
 #[macro_use]
 extern crate nom;
 
+pub mod dic;
+pub mod error;
+pub mod lattice;
 pub mod morpheme;
 pub mod tokenizer;
 
-pub mod dic;
-pub mod lattice;
+pub use error::*;
+
+#[cfg(test)]
+mod tests;
+
+pub mod prelude {
+    pub use crate::{
+        morpheme::Morpheme,
+        tokenizer::{dictionary_bytes_from_path, Mode, Tokenize, Tokenizer},
+        SudachiError, SudachiResult,
+    };
+}
diff --git a/src/main.rs b/src/main.rs
index 933d390..eda1b23 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::fs;
 use std::io::{self, BufRead, BufReader};
 use std::path::PathBuf;
@@ -5,8 +6,10 @@ use std::process;
 
 use structopt::StructOpt;
 
-use sudachi::tokenizer::Mode;
-use sudachi::tokenizer::Tokenizer;
+use sudachi::prelude::*;
+
+#[cfg(feature = "bake_dictionary")]
+const BAKED_DICTIONARY_BYTES: &[u8] = include_bytes!(env!("SUDACHI_DICT_PATH"));
 
 /// A Japanese tokenizer
 #[derive(StructOpt)]
@@ -32,11 +35,43 @@ struct Cli {
     #[structopt(short = "d", long = "debug")]
     enable_debug: bool,
 
+    // Dictionary is optional if baked in
+    /// Path to sudachi dictionary
+    #[cfg(feature = "bake_dictionary")]
+    #[structopt(short = "l", long = "dict")]
+    dictionary_path: Option<PathBuf>,
+
+    // Dictionary is not baked in, so it must be specified
+    /// Path to sudachi dictionary
+    #[cfg(not(feature = "bake_dictionary"))]
+    #[structopt(short = "l", long = "dict")]
+    dictionary_path: PathBuf,
+
     /// Prints sudachi.rs version
     #[structopt(short = "v", long = "version")]
     print_version: bool,
 }
 
+fn get_dictionary_bytes(args: &Cli) -> Cow<'static, [u8]> {
+    let dictionary_path = {
+        cfg_if::cfg_if! {
+            if #[cfg(feature="bake_dictionary")] {
+                if let Some(dictionary_path) = &args.dictionary_path {
+                    dictionary_path
+                } else {
+                    return Cow::Borrowed(BAKED_DICTIONARY_BYTES);
+                }
+            } else {
+                &args.dictionary_path
+            }
+        }
+    };
+
+    let storage_buf = dictionary_bytes_from_path(&dictionary_path)
+        .expect("Failed to get dictionary bytes from file");
+    Cow::Owned(storage_buf)
+}
+
 fn main() {
     let args = Cli::from_args();
 
@@ -45,12 +80,10 @@ fn main() {
         return;
     }
 
-    let mode = match args.mode.as_str() {
-        "A" | "a" => Mode::A,
-        "B" | "b" => Mode::B,
-        "C" | "c" => Mode::C,
-        _ => {
-            eprintln!("Invalid mode: Mode must be one of \"A\", \"B\", or \"C\" (in lower or upper case).");
+    let mode = match args.mode.as_str().parse() {
+        Ok(mode) => mode,
+        Err(err) => {
+            eprintln!("Invalid mode: {}", err);
             process::exit(1);
         }
     };
@@ -60,20 +93,24 @@ fn main() {
 
     // load and parse dictionary binary to create a tokenizer
 
-    // embed dictionary binary file
-    let bytes = include_bytes!("resources/system.dic");
-
-    let tokenizer = Tokenizer::new(bytes);
+    let dictionary_bytes = get_dictionary_bytes(&args);
+    let tokenizer = Tokenizer::from_dictionary_bytes(&dictionary_bytes)
+        .expect("Failed to create Tokenizer from dictionary bytes");
 
     // input: stdin or file
     let reader: Box<dyn BufRead> = match args.file {
         None => Box::new(BufReader::new(io::stdin())),
-        Some(input_path) => Box::new(BufReader::new(fs::File::open(input_path).unwrap())),
+        Some(input_path) => Box::new(BufReader::new(
+            fs::File::open(&input_path)
+                .unwrap_or_else(|_| panic!("Failed to open file {:?}", &input_path)),
+        )),
     };
 
     for line in reader.lines() {
-        let input = line.unwrap().to_string();
-        let morpheme_list = tokenizer.tokenize(&input, &mode, enable_debug);
+        let input = line.expect("Failed to reead line").to_string();
+        let morpheme_list = tokenizer
+            .tokenize(&input, mode, enable_debug)
+            .expect("failed to tokenize input");
 
         if wakati {
             let surface_list = morpheme_list
@@ -86,7 +123,7 @@ fn main() {
                 print!(
                     "{}\t{}\t{}",
                     morpheme.surface(),
-                    morpheme.pos().join(","),
+                    morpheme.pos().expect("Missing part of speech").join(","),
                     morpheme.normalized_form(),
                 );
                 if print_all {
diff --git a/src/morpheme.rs b/src/morpheme.rs
index 5c90f2f..5626070 100644
--- a/src/morpheme.rs
+++ b/src/morpheme.rs
@@ -1,39 +1,75 @@
+use std::fmt;
+
 use crate::dic::grammar::Grammar;
 use crate::dic::lexicon::word_infos::WordInfo;
 use crate::dic::lexicon::Lexicon;
+use crate::prelude::*;
 
+/// A morpheme (basic semantic unit of language)
 pub struct Morpheme<'a> {
     word_info: WordInfo,
     grammar: &'a Grammar<'a>,
 }
 
 impl<'a> Morpheme<'a> {
-    pub fn new(word_id: usize, grammar: &'a Grammar<'a>, lexicon: &Lexicon) -> Morpheme<'a> {
-        let word_info = lexicon.get_word_info(word_id);
-        Morpheme { word_info, grammar }
+    /// Create a new `Morpheme`
+    pub fn new(
+        word_id: usize,
+        grammar: &'a Grammar<'a>,
+        lexicon: &Lexicon,
+    ) -> SudachiResult<Morpheme<'a>> {
+        let word_info = lexicon.get_word_info(word_id)?;
+        Ok(Morpheme { word_info, grammar })
     }
 
+    /// Returns the text of morpheme.
+    ///
+    /// When the input text is normalized, some morphemes have the same surface.
     pub fn surface(&self) -> &String {
         &self.word_info.surface
     }
 
-    pub fn pos(&self) -> &Vec<String> {
-        &self
+    /// Part of speech
+    pub fn pos(&self) -> SudachiResult<&Vec<String>> {
+        let res = &self
             .grammar
             .pos_list
             .get(self.word_info.pos_id as usize)
-            .unwrap()
+            .ok_or(SudachiError::MissingPartOfSpeech)?;
+        Ok(res)
     }
 
+    /// Normalized form of morpheme
+    ///
+    /// This method returns the form normalizing inconsistent spellings and
+    /// inflected forms.
     pub fn normalized_form(&self) -> &String {
         &self.word_info.normalized_form
     }
 
+    /// Returns the reading form of morpheme.
+    ///
+    /// Returns Japanese syllabaries 'フリガナ' in katakana.
     pub fn reading_form(&self) -> &String {
         &self.word_info.reading_form
     }
 
+    /// Returns the dictionary form of morpheme.
+    ///
+    /// "Dictionary form" means a word's lemma and "終止形" in Japanese.
     pub fn dictionary_form(&self) -> &String {
         &self.word_info.dictionary_form
     }
 }
+
+impl<'a> fmt::Debug for Morpheme<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Morpheme")
+            .field("surface", self.surface())
+            .field("pos", &self.pos())
+            .field("normalized_form", self.normalized_form())
+            .field("reading_form", self.reading_form())
+            .field("dictionary_form", self.dictionary_form())
+            .finish()
+    }
+}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 9433772..feeb539 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1,43 +1,110 @@
+use std::fs::{self, File};
+use std::io::Read;
+use std::path::Path;
+use std::str::FromStr;
+
 use crate::dic::grammar::Grammar;
 use crate::dic::header::Header;
 use crate::dic::lexicon::Lexicon;
 use crate::lattice::node::Node;
 use crate::lattice::Lattice;
 use crate::morpheme::Morpheme;
+use crate::prelude::*;
+
+/// Able to tokenize Japanese text
+pub trait Tokenize {
+    /// Break text into `Morpheme`s
+    fn tokenize(&self, input: &str, mode: Mode, enable_debug: bool)
+        -> SudachiResult<Vec<Morpheme>>;
+}
 
+/// Tokenizes Japanese text
 pub struct Tokenizer<'a> {
-    bytes: &'a [u8],
     pub grammar: Grammar<'a>,
     pub lexicon: Lexicon<'a>,
 }
 
-#[derive(PartialEq)]
+/// Unit to split text
+///
+/// Some examples:
+/// ```text
+/// A：選挙/管理/委員/会
+/// B：選挙/管理/委員会
+/// C：選挙管理委員会
+///
+/// A：客室/乗務/員
+/// B：客室/乗務員
+/// C：客室乗務員
+///
+/// A：労働/者/協同/組合
+/// B：労働者/協同/組合
+/// C：労働者協同組合
+///
+/// A：機能/性/食品
+/// B：機能性/食品
+/// C：機能性食品
+/// ```
+///
+/// See [Sudachi documentation](https://github.com/WorksApplications/Sudachi#the-modes-of-splitting)
+/// for more details
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum Mode {
+    /// Short
     A,
+
+    /// Middle (similar to "word")
     B,
+
+    /// Named Entity
     C,
 }
 
-impl<'a> Tokenizer<'a> {
-    pub fn new(bytes: &'a [u8]) -> Tokenizer<'a> {
-        let mut offset = 0;
+impl FromStr for Mode {
+    type Err = &'static str;
 
-        let _header = Header::new(bytes, offset);
-        offset += Header::STORAGE_SIZE;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "A" | "a" => Ok(Mode::A),
+            "B" | "b" => Ok(Mode::B),
+            "C" | "c" => Ok(Mode::C),
+            _ => Err("Mode must be one of \"A\", \"B\", or \"C\" (in lower or upper case)."),
+        }
+    }
+}
 
-        let grammar = Grammar::new(bytes, offset);
+impl<'a> Tokenizer<'a> {
+    /// Create `Tokenizer` from the raw bytes of a Sudachi dictionary.
+    pub fn from_dictionary_bytes(dictionary_bytes: &'a [u8]) -> SudachiResult<Tokenizer<'a>> {
+        let (_rest, _header) = Header::new(&dictionary_bytes[..Header::STORAGE_SIZE])?;
+        let mut offset = Header::STORAGE_SIZE;
+
+        let grammar = Grammar::new(dictionary_bytes, offset)?;
         offset += grammar.storage_size;
 
-        let lexicon = Lexicon::new(bytes, offset);
+        let lexicon = Lexicon::new(dictionary_bytes, offset)?;
 
-        Tokenizer {
-            bytes,
-            grammar,
-            lexicon,
-        }
+        Ok(Tokenizer { grammar, lexicon })
     }
+}
 
-    pub fn tokenize(&self, input: &String, mode: &Mode, enable_debug: bool) -> Vec<Morpheme> {
+/// Return bytes of a `dictionary_path`
+pub fn dictionary_bytes_from_path<P: AsRef<Path>>(dictionary_path: P) -> SudachiResult<Vec<u8>> {
+    let dictionary_path = dictionary_path.as_ref();
+    let dictionary_stat = fs::metadata(&dictionary_path)?;
+    let mut dictionary_file = File::open(dictionary_path)?;
+    let mut dictionary_bytes = Vec::with_capacity(dictionary_stat.len() as usize);
+    dictionary_file.read_to_end(&mut dictionary_bytes)?;
+
+    Ok(dictionary_bytes)
+}
+
+impl<'a> Tokenize for Tokenizer<'a> {
+    fn tokenize(
+        &self,
+        input: &str,
+        mode: Mode,
+        enable_debug: bool,
+    ) -> SudachiResult<Vec<Morpheme>> {
         let input_bytes = input.as_bytes();
 
         // build_lattice
@@ -49,13 +116,13 @@ impl<'a> Tokenizer<'a> {
                 continue;
             }
 
-            for (word_id, end) in self.lexicon.lookup(&input_bytes, i) {
-                let (left_id, right_id, cost) = self.lexicon.get_word_param(word_id as usize);
+            for (word_id, end) in self.lexicon.lookup(&input_bytes, i)? {
+                let (left_id, right_id, cost) = self.lexicon.get_word_param(word_id as usize)?;
                 let node = Node::new(left_id, right_id, cost, word_id);
-                lattice.insert(i, end, node);
+                lattice.insert(i, end, node)?;
             }
         }
-        lattice.connect_eos_node();
+        lattice.connect_eos_node()?;
 
         // lattice dump
         if enable_debug {
@@ -67,7 +134,7 @@ impl<'a> Tokenizer<'a> {
                     for l_node in &lattice.end_lists[r_node.begin] {
                         let connect_cost = self
                             .grammar
-                            .get_connect_cost(l_node.right_id, r_node.left_id);
+                            .get_connect_cost(l_node.right_id, r_node.left_id)?;
                         let cost = l_node.total_cost + connect_cost as i32;
                         print!("{} ", cost);
                     }
@@ -78,24 +145,26 @@ impl<'a> Tokenizer<'a> {
             println!("===");
         };
 
-        let node_list = lattice.get_best_path();
+        let node_list = lattice.get_best_path()?;
 
         let mut word_id_list = Vec::new();
-        if *mode == Mode::C {
+        if mode == Mode::C {
             word_id_list = node_list
                 .iter()
-                .map(|node| node.word_id.unwrap() as usize)
-                .collect::<Vec<_>>();
+                .map(|node| node.word_id.map(|x| x as usize))
+                .collect::<Option<Vec<_>>>()
+                .ok_or_else(|| SudachiError::MissingWordId)?;
         } else {
             for node in &node_list {
-                let node_word_id = node.word_id.unwrap() as usize;
-                let word_ids = match *mode {
-                    Mode::A => self.lexicon.get_word_info(node_word_id).a_unit_split,
-                    Mode::B => self.lexicon.get_word_info(node_word_id).b_unit_split,
-                    _ => panic!(),
+                let node_word_id =
+                    node.word_id.ok_or_else(|| SudachiError::MissingWordId)? as usize;
+                let word_ids = match mode {
+                    Mode::A => self.lexicon.get_word_info(node_word_id)?.a_unit_split,
+                    Mode::B => self.lexicon.get_word_info(node_word_id)?.b_unit_split,
+                    _ => unreachable!(),
                 };
 
-                if (word_ids.len() == 0) | (word_ids.len() == 1) {
+                if word_ids.is_empty() | (word_ids.len() == 1) {
                     word_id_list.push(node_word_id);
                 } else {
                     for word_id in word_ids {
@@ -105,11 +174,9 @@ impl<'a> Tokenizer<'a> {
             }
         };
 
-        let morpheme_list = word_id_list
+        word_id_list
             .iter()
             .map(|word_id| Morpheme::new(*word_id, &self.grammar, &self.lexicon))
-            .collect::<Vec<_>>();
-
-        morpheme_list
+            .collect::<SudachiResult<Vec<_>>>()
     }
 }


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
new file mode 100644
index 0000000..d0c9752
--- /dev/null
+++ b/src/tests.rs
@@ -0,0 +1,95 @@
+//! Crate tests
+
+use std::env;
+
+use crate::prelude::*;
+
+lazy_static! {
+    static ref DICTIONARY_BYTES: Vec<u8> = {
+        let dictionary_path = env::var_os("SUDACHI_DICT_PATH").expect("Must set env var SUDACHI_DICT_PATH with path to Sudachi dictionary (relative to current dir)");
+        let dictionary_bytes = dictionary_bytes_from_path(dictionary_path)
+            .expect("Failed to read dictionary from path");
+        dictionary_bytes
+    };
+    static ref TOKENIZER: Tokenizer<'static> = Tokenizer::from_dictionary_bytes(&DICTIONARY_BYTES)
+        .expect("Failed to create Tokenizer for tests");
+}
+
+/// Expected chunks for a text in a given mode
+#[derive(Debug, Clone)]
+struct ChunkExpectation<'a> {
+    a: &'a [&'a str],
+    b: &'a [&'a str],
+    c: &'a [&'a str],
+}
+
+/// Get text chunks from text
+fn tokenized_chunks(text: &str, mode: Mode) -> Vec<String> {
+    let tokens = TOKENIZER
+        .tokenize(text, mode, false)
+        .expect("Failed to get tokens");
+    tokens
+        .iter()
+        .map(|tok| tok.surface().clone())
+        .collect::<Vec<String>>()
+}
+
+fn assert_tokenized_chunk_modes(text: &str, expected_chunks: ChunkExpectation) {
+    assert_eq!(
+        tokenized_chunks(text, Mode::A),
+        expected_chunks.a,
+        "failed for mode A, text={:?}",
+        text
+    );
+    assert_eq!(
+        tokenized_chunks(text, Mode::B),
+        expected_chunks.b,
+        "failed for mode B, text={:?}",
+        text
+    );
+    assert_eq!(
+        tokenized_chunks(text, Mode::C),
+        expected_chunks.c,
+        "failed for mode C, text={:?}",
+        text
+    );
+}
+
+#[test]
+fn chunks() {
+    assert_tokenized_chunk_modes(
+        "選挙管理委員会",
+        ChunkExpectation {
+            a: &["選挙", "管理", "委員", "会"],
+            b: &["選挙", "管理", "委員会"],
+            c: &["選挙管理委員会"],
+        },
+    );
+
+    assert_tokenized_chunk_modes(
+        "客室乗務員",
+        ChunkExpectation {
+            a: &["客室", "乗務", "員"],
+            b: &["客室", "乗務員"],
+            c: &["客室乗務員"],
+        },
+    );
+
+    assert_tokenized_chunk_modes(
+        "労働者協同組合",
+        ChunkExpectation {
+            a: &["労働", "者", "協同", "組合"],
+            b: &["労働者", "協同", "組合"],
+            c: &["労働者協同組合"],
+        },
+    );
+
+    assert_tokenized_chunk_modes(
+        "機能性食品",
+        ChunkExpectation {
+            a: &["機能", "性", "食品"],
+            b: &["機能性", "食品"],
+            c: &["機能性食品"],
+        },
+    );
+}