Repository: megagonlabs/ginza (language: python)

Issue:
final preparation for v5.1.0



Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 7b799a3..59fa77e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,4 +20,5 @@
 __pycache__/
 *.pyc
 *.egg-info/
-.DS_Store
\ No newline at end of file
+.DS_Store
+.eggs
\ No newline at end of file
diff --git a/README.md b/README.md
index 379898e..e0d725b 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
 
 An Open Source Japanese NLP Library, based on Universal Dependencies
 
-***Please read the [Important changes](#ginza-500) before you upgrade GiNZA.***
+***Please read the [Important changes](#ginza-510) before you upgrade GiNZA.***
 
 [日本語ページはこちら](https://megagonlabs.github.io/ginza/)
 
@@ -36,14 +36,14 @@ The GiNZA v5 Transformers model (ja_ginza_electra) is trained by using Hugging F
 
 ## Training Datasets
 
-### UD Japanese BCCWJ v2.6
-The parsing model of GiNZA v4 is trained on a part of
-[UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) v2.6
+### UD Japanese BCCWJ r2.8
+The parsing model of GiNZA v5 is trained on a part of
+[UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) r2.8
 ([Omura and Asahara:2018](https://www.aclweb.org/anthology/W18-6014/)).
 This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs.
 
 ### GSK2014-A (2019) BCCWJ edition
-The named entity recognition model of GiNZA v4 is trained on a part of
+The named entity recognition model of GiNZA v5 is trained on a part of
 [GSK2014-A](https://www.gsk.or.jp/catalog/gsk2014-a/) (2019) BCCWJ edition
 ([Hashimoto, Inui, and Murakami:2008](https://www.anlp.jp/proceedings/annual_meeting/2010/pdf_dir/C4-4.pdf)).
 We use two of the named entity label systems, both
@@ -52,7 +52,7 @@ and extended [OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNote
 This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs.
 
 ### mC4
-The GiNZA v5 Transformers model (ja-ginza-electra) is trained by using [transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator) which is pretrained on more than 200 million Japanese sentences extracted from [mC4](https://huggingface.co/datasets/mc4).
+The GiNZA v5 Transformers model (ja_ginza_electra) is trained by using [transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator) which is pretrained on more than 200 million Japanese sentences extracted from [mC4](https://huggingface.co/datasets/mc4).
 
 Contains information from mC4 which is made available under the ODC Attribution License.
 ```
@@ -74,19 +74,19 @@ Please also see the Development Environment section below.
 ### Runtime set up
 
 #### 1. Install GiNZA NLP Library with Transformer-based Model
-Uninstall previous version:
+Uninstall previous version of ginza and ja_ginza_electra packages:
 ```console
-$ pip uninstall ginza ja-ginza
+$ pip uninstall ginza ja_ginza_elecrta
 ```
-Then, install the latest version of `ginza` and `ja-ginza-electra`:
+Then, install the latest version of `ginza` and `ja_ginza_electra`:
 ```console
-$ pip install -U ginza ja-ginza-electra
+$ pip install -U ginza ja_ginza_electra
 ```
 
-The package of `ja-ginza-electra` does not include `pytorch_model.bin` due to PyPI's archive size restrictions.
+The package of `ja_ginza_electra` does not include `pytorch_model.bin` due to PyPI's archive size restrictions.
 This large model file will be automatically downloaded at the first run time, and the locally cached file will be used for subsequent runs.
 
-If you need to install `ja-ginza-electra` along with `pytorch_model.bin` at the install time, you can specify direct link for GitHub release archive as follows:
+If you need to install `ja_ginza_electra` along with `pytorch_model.bin` at the install time, you can specify direct link for GitHub release archive as follows:
 ```console
 $ pip install -U ginza https://github.com/megagonlabs/ginza/releases/download/latest/ja_ginza_electra-latest-with-model.tar.gz
 ```
@@ -96,14 +96,16 @@ If you hope to accelarate the transformers-based models by using GPUs with CUDA
 pip install -U "spacy[cuda110]"
 ```
 
+And you need to install a version of pytorch that is consistent with the CUDA version.
+
 #### 2. Install GiNZA NLP Library with Standard Model
 Uninstall previous version:
 ```console
-$ pip uninstall ginza ja-ginza
+$ pip uninstall ginza ja_ginza
 ```
-Then, install the latest version of `ginza` and `ja-ginza`:
+Then, install the latest version of `ginza` and `ja_ginza`:
 ```console
-$ pip install -U ginza ja-ginza
+$ pip install -U ginza ja_ginza
 ```
 
 ### Execute ginza command
@@ -206,21 +208,52 @@ nlp = spacy.load('ja_ginza_electra')
 doc = nlp('銀座でランチをご一緒しましょう。')
 for sent in doc.sents:
     for token in sent:
-        print(token.i, token.orth_, token.lemma_, token.pos_, token.tag_, token.dep_, token.head.i)
+        print(
+          token.i,
+          token.orth_,
+          token.lemma_,
+          token.norm_
+          token.morph.get(“Reading”),
+          token.pos_,
+          token.morph.get(“Inflection”),
+          token.tag_,
+          token.dep_,
+          token.head.i,
+        )
     print('EOS')
 ```
 
 ### User Dictionary
 The user dictionary files should be set to `userDict` field of `sudachi.json` in the installed package directory of`ja_ginza_dict` package.
 
-Please read the official documents to compile user dictionaries with `sudachipy` command.  
-[SudachiPy - User defined Dictionary](https://github.com/WorksApplications/SudachiPy#user-defined-dictionary)  
+Please read the official documents to compile user dictionaries with `sudachipy` command.
+[SudachiPy - User defined Dictionary](https://github.com/WorksApplications/SudachiPy#user-defined-dictionary)
 [Sudachi User Dictionary Construction (Japanese Only)](https://github.com/WorksApplications/Sudachi/blob/develop/docs/user_dict.md)
 
 ## Releases
 
 ### version 5.x
 
+#### ginza-5.1.0
+- 2021-12-10, Euclase
+- Important changes
+  - Upgrade: spaCy v3.2 and Sudachi.rs(SudachiPy v0.6.2)
+  - Change token information fields #208 #209
+    - `doc.user_data[“reading_forms”][token.i]` -> `token.morph.get(“Reading”)`
+    - `doc.user_data[“inflections”][token.i]` -> `token.morph.get(“Inflection”)`
+    - `force_using_normalized_form_as_lemma(True)` -> `token.norm_`
+  - All spaCy models, including non-Japanese, are now available with the ginza command #217
+    - Download and analyze the model at once by specifying the model name in the following form #219
+    - `ginza -m en_core_web_md`
+  - Change `ginza --require_gpu`  and `ginza -g` to take a `gpu_id` argument
+    - The default `gpu_id` value is `-1` which uses only CPUs
+  - `ginza -f json` option always analyze the line which starts with `#` regardless the option value of `-c`. #215
+- Improvements
+  - Batch analysis processing speeds up by 50-60% in GPU environment and 10-40% in CPU environment
+  - Improved processing efficiency of parallel execution options (`ginza -p {n_process}` and `ginzame`) of ginza command #204
+  - add tests #198 #210 #214
+  - add benchmark #207 #220
+
 #### ginza-5.0.3
 - 2021-10-15
 - Bug fix
@@ -433,8 +466,8 @@ with open('sample2.pickle', 'wb') as f:
     - `sudachi`: SudachiPy's morpheme instance (or its list when then tokens are gathered by JapaneseCorrector)
 - Performance improvements
   - Tokenizer
-    - Use latest SudachiDict (SudachiDict_core-20190927.tar.gz) 
-    - Use Cythonized SudachiPy (v0.4.0) 
+    - Use latest SudachiDict (SudachiDict_core-20190927.tar.gz)
+    - Use Cythonized SudachiPy (v0.4.0)
   - Dependency parser
     - Apply `spacy pretrain` command to capture the language model from UD-Japanese BCCWJ, UD_Japanese-PUD and KWDLC.
     - Apply multitask objectives by using `-pt 'tag,dep'` option of `spacy train`
@@ -495,3 +528,16 @@ The analysis model of GiNZA is trained by `spacy train` command.
 ```console
 $ python -m spacy train ja ja_ginza-4.0.0 corpus/ja_ginza-ud-train.json corpus/ja_ginza-ud-dev.json -b ja_vectors_chive_mc90_35k/ -ovl 0.3 -n 100 -m meta.json.ginza -V 4.0.0
 ```
+
+## Run tests
+Ginza uses the pytest framework for testing, and you can run the tests via `setup.py` without install test requirements explicitly.
+Some tests depends on the ginza default models (`ja-ginza`, `ja-ginza-electra`), so install them before the tests is needed.
+
+```console
+pip install ja-ginza ja-ginza-electra
+pip install -e .
+# full test
+python setup.py test
+# test single file
+python setup.py test --addopts ginza/tests/test_analyzer.py
+```
diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index 02afc85..449b177 100644
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -6,12 +6,15 @@
 REPEAT = 5
 BATCH_SIZE = 128
 
-assert len(sys.argv) == 1 or len(sys.argv) == 2 and sys.argv[1] == "-g", "Usage: python {sys.argv[0]} {sys.argv[1]} [-g]"
-require_gpu = len(sys.argv) == 2 and sys.argv[1] == "-g"
-if require_gpu:
+assert len(sys.argv) >= 2, "Usage: python {sys.argv[0]} [-g] model_name1 [model_name2 [...]]"
+if sys.argv[1] == "-g":
+    require_gpu = True
     device = "GPU"
+    model_names = sys.argv[2:]
 else:
+    require_gpu = False
     device = "CPU"
+    model_names = sys.argv[1:]
 
 sents = [_.rstrip("\n") for _ in sys.stdin]
 
@@ -21,105 +24,70 @@
 print("timestamp                 ", "[msec]", "device", 'procedure description', sep="\t", file=sys.stderr)
 start = datetime.now()
 prev = start
-print(start, 0, f'benchmark started with {len(sents)} sentences', sep="\t", file=sys.stderr)
+print(start, 0, f"benchmark started with {len(sents)} sentences", sep="\t", file=sys.stderr)
 
 import spacy
 if require_gpu:
     spacy.require_gpu()
 lap = datetime.now()
 dur = int((lap - prev).total_seconds() * 1000)
-results[f"import spacy"] = [dur]
 print(lap, dur, device, 'import spacy', sep="\t", file=sys.stderr)
 prev = lap
 
-nlp = spacy.load("ja_ginza")
-lap = datetime.now()
-dur = int((lap - prev).total_seconds() * 1000)
-results[f"spacy.load('ja_ginza')"] = [dur]
-print(lap, dur, device, f'spacy.load("ja_ginza")', sep="\t", file=sys.stderr)
-prev = lap
-
-results[f"ja_ginza->nlp(batch={BATCH_SIZE})"] = []
-for repeat in range(1, REPEAT + 1):
-    for _ in range((len(sents) - 1) // BATCH_SIZE + 1):
-        doc = nlp("\n".join(sents[_ * BATCH_SIZE:(_ + 1) * BATCH_SIZE]))
-    lap = datetime.now()
-    dur = int((lap - prev).total_seconds() * 1000)
-    results[f"ja_ginza->nlp(batch={BATCH_SIZE})"].append(dur / len(sents))
-    print(
-        lap,
-        dur,
-        device,
-        f'#{repeat} ja_ginza->nlp(batch={BATCH_SIZE}): {dur / len(sents):.03f}[msec/sent]',
-        sep="\t", file=sys.stderr,
-    )
-    prev = lap
-
-results[f"ja_ginza->nlp(batch=1)"] = []
-for repeat in range(1, REPEAT + 1):
-    for sent in sents:
-        doc = nlp(sent)
-    lap = datetime.now()
-    dur = int((lap - prev).total_seconds() * 1000)
-    results[f"ja_ginza->nlp(batch=1)"].append(dur / len(sents))
-    print(
-        lap,
-        dur,
-        device,
-        f'#{repeat} ja_ginza->nlp(batch=1):   {dur / len(sents):.03f}[msec/sent]',
-        sep="\t", file=sys.stderr,
-    )
-    prev = lap
-
-nlp = spacy.load("ja_ginza_electra")
-lap = datetime.now()
-dur = int((lap - prev).total_seconds() * 1000)
-results[f"spacy.load('ja_ginza_electra')"] = [dur]
-print(lap, dur, device, f'spacy.load("ja_ginza_electra")', sep="\t", file=sys.stderr)
-prev = lap
-
-results[f"ja_ginza_electra->nlp(batch={BATCH_SIZE})"] = []
-for repeat in range(1, REPEAT + 1):
-    for _ in range((len(sents) - 1) // BATCH_SIZE + 1):
-        doc = nlp("\n".join(sents[_ * BATCH_SIZE:(_ + 1) * BATCH_SIZE]))
+for model_name in model_names:
+    results = {}
+    nlp = spacy.load(model_name)
     lap = datetime.now()
     dur = int((lap - prev).total_seconds() * 1000)
-    results[f"ja_ginza_electra->nlp(batch={BATCH_SIZE})"].append(dur / len(sents))
-    print(
-        lap,
-        dur,
-        device,
-        f'#{repeat} ja_ginza_electra->nlp(batch={BATCH_SIZE}): {dur / len(sents):.03f}[msec/sent]',
-        sep="\t", file=sys.stderr,
-    )
+    results[f"spacy.load()"] = [dur]
+    print(lap, dur, device, f"spacy.load({model_name})", sep="\t", file=sys.stderr)
     prev = lap
 
-results[f"ja_ginza_electra->nlp(batch=1)"] = []
-for repeat in range(1, REPEAT + 1):
-    for sent in sents:
-        doc = nlp(sent)
-    lap = datetime.now()
-    dur = int((lap - prev).total_seconds() * 1000)
-    results[f"ja_ginza_electra->nlp(batch=1)"].append(dur / len(sents))
-    print(
-        lap,
-        dur,
-        device,
-        f'#{repeat} ja_ginza_electra->nlp(batch=1):   {dur / len(sents):.03f}[msec/sent]',
-        sep="\t", file=sys.stderr,
+    results[f"nlp.pipe(batch={BATCH_SIZE})"] = []
+    for repeat in range(1, REPEAT + 1):
+        for _ in range((len(sents) - 1) // BATCH_SIZE + 1):
+            docs = nlp.pipe(sents[_ * BATCH_SIZE:(_ + 1) * BATCH_SIZE])
+            for doc in docs:
+                len(doc)
+        lap = datetime.now()
+        dur = int((lap - prev).total_seconds() * 1000)
+        results[f"nlp.pipe(batch={BATCH_SIZE})"].append(dur / len(sents))
+        print(
+            lap,
+            dur,
+            device,
+            f"#{repeat} {model_name}->nlp.pipe(batch={BATCH_SIZE}): {dur / len(sents):.03f}[msec/sent]",
+            sep="\t", file=sys.stderr,
+        )
+        prev = lap
+
+    results[f"nlp(batch=1)"] = []
+    for repeat in range(1, REPEAT + 1):
+        for sent in sents:
+            doc = nlp(sent)
+            len(doc)
+        lap = datetime.now()
+        dur = int((lap - prev).total_seconds() * 1000)
+        results[f"nlp(batch=1)"].append(dur / len(sents))
+        print(
+            lap,
+            dur,
+            device,
+            f"#{repeat} {model_name}->nlp(batch=1):   {dur / len(sents):.03f}[msec/sent]",
+            sep="\t", file=sys.stderr,
+        )
+        prev = lap
+
+    dur = int((lap - start).total_seconds() * 1000)
+    print(lap, dur, device, model_name, 'finished', sep="\t", file=sys.stderr)
+
+    for k, v in results.items():
+        l = sorted(v)
+        results[k] = l[len(l) // 2]
+
+    json.dump(
+        {"model": model_name, "device": device, "results": results},
+        sys.stdout,
+        ensure_ascii=False,
     )
-    prev = lap
-
-dur = int((lap - start).total_seconds() * 1000)
-print(lap, dur, device, 'total', sep="\t", file=sys.stderr)
-
-for k, v in results.items():
-    l = sorted(v)
-    results[k] = l[len(l) // 2]
-
-json.dump(
-    {"device": device, "results": results},
-    sys.stdout,
-    ensure_ascii=False,
-)
-print()
+    print()
diff --git a/benchmark/run_benchmark.sh b/benchmark/run_benchmark.sh
deleted file mode 100755
index 23dec0c..0000000
--- a/benchmark/run_benchmark.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-set -e
-cat gsd/dev.txt gsd/test.txt | python benchmark.py -g
-cat gsd/dev.txt gsd/test.txt | python benchmark.py
diff --git a/benchmark/run_benchmark_ginza.sh b/benchmark/run_benchmark_ginza.sh
new file mode 100755
index 0000000..daa033a
--- /dev/null
+++ b/benchmark/run_benchmark_ginza.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+cat gsd/dev.txt gsd/test.txt | python benchmark.py -g ja_ginza ja_ginza_electra
+cat gsd/dev.txt gsd/test.txt | python benchmark.py ja_ginza ja_ginza_electra
diff --git a/benchmark/run_benchmark_spacy.sh b/benchmark/run_benchmark_spacy.sh
new file mode 100755
index 0000000..9f12c19
--- /dev/null
+++ b/benchmark/run_benchmark_spacy.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+cat gsd/dev.txt gsd/test.txt | python benchmark.py -g ja_core_news_md ja_core_news_trf
+cat gsd/dev.txt gsd/test.txt | python benchmark.py ja_core_news_md ja_core_news_trf
diff --git a/benchmark/setup_benchmark.sh b/benchmark/setup_benchmark.sh
index 946e623..a1c39eb 100755
--- a/benchmark/setup_benchmark.sh
+++ b/benchmark/setup_benchmark.sh
@@ -9,3 +9,4 @@ echo '=== CUDA Related Installation Steps ==='
 echo 'The pytorch should be installed with cuda support. See https://pytorch.org/get-started/previous-versions/#linux-and-windows-1'
 echo 'Also you need to install spacy with appropriate cuda specifier as `pip install -U spacy[cudaXXX]`. See https://spacy.io/usage#gpu'
 echo 'And then, install GiNZA as `pip install -U ginza ja-ginza ja-ginza-electra`.'
+echo 'To evaluate the performance of spaCy official models, install models as `python -m spacy download ja_core_news_md ; python -m spacy download ja_core_news_trf`.'
diff --git a/config/ja_ginza.meta.json b/config/ja_ginza.meta.json
index 7c43873..d276a14 100644
--- a/config/ja_ginza.meta.json
+++ b/config/ja_ginza.meta.json
@@ -1,7 +1,7 @@
 {
   "lang":"ja",
   "name":"ginza",
-  "version":"5.0.0",
+  "version":"5.1.0",
   "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019). Assigns word2vec token vectors. Components: tok2vec, parser, ner, morphologizer, atteribute_ruler, compound_splitter, bunsetu_recognizer.",
   "author":"Megagon Labs Tokyo.",
   "email":"ginza@megagon.ai",
@@ -34,7 +34,7 @@
     }
   ],
   "parent_package":"spacy",
-  "spacy_version":">=3.0.6,<3.2.0",
+  "spacy_version":">=3.2.0,<3.3.0",
   "pipeline":[
     "tok2vec",
     "parser",
@@ -57,8 +57,8 @@
     "attribute_ruler"
   ],
   "requirements":[
-    "sudachipy>=0.5.2,<0.6.0",
-    "sudachidict_core>=20210608",
-    "ginza>=5.0.0,<5.1.0"
+    "sudachipy>=0.6.2,<0.7.0",
+    "sudachidict_core>=20210802",
+    "ginza>=5.1.0,<5.2.0"
   ]
 }
diff --git a/config/ja_ginza_electra.analysis.cfg b/config/ja_ginza_electra.analysis.cfg
index 42fa75d..17666e9 100644
--- a/config/ja_ginza_electra.analysis.cfg
+++ b/config/ja_ginza_electra.analysis.cfg
@@ -25,6 +25,7 @@ split_mode = "C"
 
 [components.attribute_ruler]
 factory = "attribute_ruler"
+scorer = {"@scorers":"spacy.attribute_ruler_scorer.v1"}
 validate = false
 
 [components.bunsetu_recognizer]
@@ -37,6 +38,9 @@ split_mode = null
 
 [components.morphologizer]
 factory = "morphologizer"
+extend = true
+overwrite = true
+scorer = {"@scorers":"spacy.morphologizer_scorer.v1"}
 
 [components.morphologizer.model]
 @architectures = "spacy.Tagger.v1"
@@ -52,6 +56,7 @@ upstream = "*"
 factory = "ner"
 incorrect_spans_key = null
 moves = null
+scorer = {"@scorers":"spacy.ner_scorer.v1"}
 update_with_oracle_cut_size = 100
 
 [components.ner.model]
@@ -74,6 +79,7 @@ factory = "parser"
 learn_tokens = false
 min_action_freq = 30
 moves = null
+scorer = {"@scorers":"spacy.parser_scorer.v1"}
 update_with_oracle_cut_size = 100
 
 [components.parser.model]
@@ -97,14 +103,17 @@ max_batch_items = 4096
 set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
 
 [components.transformer.model]
-@architectures = "ginza-transformers.TransformerModel.v1"
-name = "megagonlabs/transformers-ud-japanese-electra-base-ginza"
+@architectures = "spacy-transformers.TransformerModel.v3"
+name = "megagonlabs/transformers-ud-japanese-electra-base-ginza-510"
+mixed_precision = false
 
 [components.transformer.model.get_spans]
 @span_getters = "spacy-transformers.strided_spans.v1"
 window = 128
 stride = 96
 
+[components.transformer.model.grad_scaler_config]
+
 [components.transformer.model.tokenizer_config]
 use_fast = false
 tokenizer_class = "sudachitra.tokenization_electra_sudachipy.ElectraSudachipyTokenizer"
@@ -119,6 +128,8 @@ word_form_type = "dictionary_and_surface"
 split_mode = "A"
 dict_type = "core"
 
+[components.transformer.model.transformer_config]
+
 [corpora]
 
 [corpora.dev]
@@ -149,8 +160,8 @@ max_epochs = 0
 max_steps = 50000
 eval_frequency = 200
 frozen_components = []
-before_to_disk = null
 annotating_components = []
+before_to_disk = null
 
 [training.batcher]
 @batchers = "spacy.batch_by_padded.v1"
diff --git a/config/ja_ginza_electra.cfg b/config/ja_ginza_electra.cfg
index 376ce93..e9d8ff4 100644
--- a/config/ja_ginza_electra.cfg
+++ b/config/ja_ginza_electra.cfg
@@ -37,6 +37,9 @@ split_mode = null
 
 [components.morphologizer]
 factory = "morphologizer"
+extend = true
+overwrite = true
+scorer = {"@scorers":"spacy.morphologizer_scorer.v1"}
 
 [components.morphologizer.model]
 @architectures = "spacy.Tagger.v1"
@@ -52,6 +55,7 @@ upstream = "*"
 factory = "ner"
 incorrect_spans_key = null
 moves = null
+scorer = {"@scorers":"spacy.ner_scorer.v1"}
 update_with_oracle_cut_size = 100
 
 [components.ner.model]
@@ -74,6 +78,7 @@ factory = "parser"
 learn_tokens = false
 min_action_freq = 30
 moves = null
+scorer = {"@scorers":"spacy.parser_scorer.v1"}
 update_with_oracle_cut_size = 100
 
 [components.parser.model]
@@ -92,19 +97,22 @@ pooling = {"@layers":"reduce_mean.v1"}
 upstream = "*"
 
 [components.transformer]
-factory = "transformer"
+factory = "transformer_custom"
 max_batch_items = 4096
 set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
 
 [components.transformer.model]
-@architectures = "ginza-transformers.TransformerModel.v1"
+@architectures = "spacy-transformers.TransformerModel.v3"
 name = "megagonlabs/transformers-ud-japanese-electra-base-discriminator"
+mixed_precision = false
 
 [components.transformer.model.get_spans]
 @span_getters = "spacy-transformers.strided_spans.v1"
 window = 128
 stride = 96
 
+[components.transformer.model.grad_scaler_config]
+
 [components.transformer.model.tokenizer_config]
 use_fast = false
 tokenizer_class = "sudachitra.tokenization_electra_sudachipy.ElectraSudachipyTokenizer"
@@ -119,6 +127,8 @@ word_form_type = "dictionary_and_surface"
 split_mode = "A"
 dict_type = "core"
 
+[components.transformer.model.transformer_config]
+
 [corpora]
 
 [corpora.dev]
diff --git a/config/ja_ginza_electra.meta.json b/config/ja_ginza_electra.meta.json
index 5469971..658d887 100644
--- a/config/ja_ginza_electra.meta.json
+++ b/config/ja_ginza_electra.meta.json
@@ -1,7 +1,7 @@
 {
   "lang":"ja",
   "name":"ginza_electra",
-  "version":"5.0.0",
+  "version":"5.1.0",
   "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019) + transformers-ud-japanese-electra--base. Components: transformer, parser, atteribute_ruler, ner, morphologizer, compound_splitter, bunsetu_recognizer.",
   "author":"Megagon Labs Tokyo.",
   "email":"ginza@megagon.ai",
@@ -41,7 +41,7 @@
     }
   ],
   "parent_package":"spacy",
-  "spacy_version":">=3.0.6,<3.2.0",
+  "spacy_version":">=3.2.0,<3.3.0",
   "pipeline":[
     "transformer",
     "parser",
@@ -64,10 +64,10 @@
     "attribute_ruler"
   ],
   "requirements":[
-    "sudachipy>=0.5.2,<0.6.0",
-    "sudachidict_core>=20210608",
-    "sudachitra>=0.1.5,<0.2.0",
-    "ginza-transformers>=0.3.1,<1.0.0",
-    "ginza>=5.0.0,<5.1.0"
+    "sudachipy>=0.6.2,<0.7.0",
+    "sudachidict_core>=20210802",
+    "sudachitra>=0.1.6,<0.2.0",
+    "ginza-transformers>=0.4.0,<0.5.0",
+    "ginza>=5.1.0,<5.2.0"
   ]
 }
diff --git a/docs/command_line_tool.md b/docs/command_line_tool.md
index 0e72e58..b018eaf 100644
--- a/docs/command_line_tool.md
+++ b/docs/command_line_tool.md
@@ -23,8 +23,8 @@ $ ginza
 
 `ginzame`コマンドでオープンソース形態素解析エンジン [MeCab](https://taku910.github.io/mecab/) の`mecab`コマンドに近い形式で解析結果を出力することができます。
 `ginzame`コマンドは形態素解析処理のみをマルチプロセスで高速に実行します。
-このコマンドと`mecab`の出力形式の相違点として、 
-最終フィールド（発音）が常に`*`となることに注意して下さい。
+このコマンドと`mecab`の出力形式の相違点として、最終フィールド（発音）が常に`*`となること、
+ginza の split_mode はデフォルトが `C` なので unidic 相当の単語分割を得るためには `-s A` を指定する必要があることに注意して下さい。
 ```console
 $ ginzame
 銀座でランチをご一緒しましょう。
@@ -41,6 +41,49 @@ EOS
 
 ```
 
+## OPTIONS
+`ginza`コマンドでは以下のオプションを指定することができます。
+`ginzame`コマンドでは `--split-mode` `--hash-comment` `output-path` `--use-normalized-form` `--parallel` オプションが利用可能です。
+
+- `--model-path <string>`, `-b <string>`
+    `spacy.language.Language` 形式の学習済みモデルが保存されたディレクトリを指定します。
+    `--ensure-model` オプションと同時に指定することはできません。
+- `--ensure-model <string>`, `-m <string>`
+    ginza および spaCy が公開している学習済みモデル名を指定します。`--model-path` オプションと同時に指定することはできません。次の値のいずれかを指定できます。
+        - `ja_ginza`, `ja_ginza_electra`
+        - [spaCy Models & Languages](https://spacy.io/usage/models)で公開されている日本語以外を含む全ての言語のモデル (例: en_core_web_md)
+    使用するモデルに応じて、事前に `pip install ja-ginza-electra` のようにパッケージをダウンロードする必要があります。
+    `--model-path`, `--ensure-model` のどちらも指定されない場合には `ja_ginza_electra`、`ja_ginza` の順の優先度でロード可能なモデルを利用します。
+- `--split-mode <string>`, `-s <string>`
+     複合名詞の分割モードを指定します。モードは [sudachi](https://github.com/WorksApplications/Sudachi#the-modes-of-splitting) に準拠し、`A`、`B`、`C`のいずれかを指定できます。`ginza`コマンドのデフォルト値は `C`、`ginzame`コマンドのデフォルト値はMeCab UniDicに近い `A` です。
+     `A`が分割が最も短く複合名詞が UniDic 短単位まで分割され、 `C` では固有名詞が抽出されます。`B` は二つの中間の単位に分割されます。
+- `--hash-comment <string>`, `-c <string>`
+    行頭が `#` から始まる行を解析対象とするかのモードを指定します。次の値のいずれかを指定できます。
+        - `print`
+            解析対象とはしないが、解析結果には入力をそのまま出力します。
+        - `skip`
+            解析対象とせず、解析結果にも出力しません。
+        - `analyze`
+            `#` から始まる行についても解析を行い、結果を出力します。ただし`-f json`が指定されている場合は `-c`の指定に依らず常に`analyze`が適用されます。
+    デフォルト値は `print` です。
+- `--output-path <string>`, `-o <string>`
+    解析結果を出力するファイルのパスを指定します。指定しない場合には標準出力に解析結果が出力されます。
+- `--output-format <string>`, `-f <string>`
+    [解析結果のフォーマット](#出力形式の指定)を指定します。次の値のいずれかを指定できます。
+        - `0`, `conllu`
+        - `1`, `cabocha`
+        - `2`, `mecab`
+        - `3`, `json`
+    デフォルト値は `conllu` です。
+- `--require-gpu <int>`, `-g <int>`
+    引数で指定されたgpu_idのGPUを使用して解析を行います。引数に-1を指定(デフォルト)するとCPUを使用します。ただし、[spaCyおよびcupyの制約](https://github.com/explosion/spaCy/issues/5507)から、`--require-gpu`は`--parallel`と同時に指定できません。
+- `--use-normalized-form`, `-n`
+    `-f conllu`のlemmaフィールドに [sudachi](https://github.com/WorksApplications/Sudachi#normalized-form) を使用するためのブールスイッチ。
+- `--disable-sentencizer`, `-d`
+    `ja_ginza`、 `ja_ginza_electra` モデル利用時に[disable_sentencizer](https://github.com/megagonlabs/ginza/blob/develop/ginza/disable_sentencizer.py)を有効化するブールスイッチ。
+- `--parallel <int>`, `-p <int>`
+    並列実行するプロセス数を指定します。0 を指定すると cpu コア数分のプロセスを起動します。デフォルト値は1です。
+
 ## 出力形式の指定
 
 ### JSON
@@ -79,7 +122,7 @@ $ ginza -f json
 
 日本語係り受け解析器 [CaboCha](https://taku910.github.io/cabocha/) の`cabocha -f1`のラティス形式に近い解析結果を出力する場合は
 `ginza -f 1` または `ginza -f cabocha` を実行して下さい。
-このオプションと`cabocha -f1`の出力形式の相違点として、 
+このオプションと`cabocha -f1`の出力形式の相違点として、
 スラッシュ記号`/`に続く`func_index`フィールドが常に自立語の終了位置（機能語があればその開始位置に一致）を示すこと、
 機能語認定基準が一部異なること、
 に注意して下さい。
@@ -110,4 +153,4 @@ EOS
 
 `ginza -f mecab`とそのエイリアスである`ginzame`以外で`-p NUM_PROCESS`オプションを使用する場合は、
 実行環境の空きメモリ容量が十分あることを事前に確認してください。
-マルチプロセス実行では1プロセスあたり約130MBのメモリが必要です。(今後のリリースで改善予定)
+マルチプロセス実行では1プロセスあたり`ja_ginza`で数百MB、`ja_ginza_electra`で数GBのメモリが必要です。
diff --git a/docs/index.md b/docs/index.md
index 12a86cd..82d0079 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -8,23 +8,31 @@
 
 ## What's new!
 
-- `spaCy v3`に対応した`GiNZA v5`をリリースしました。
-- transformersモデルを採用して飛躍的に精度を向上した解析モデルパッケージ`ja-ginza-electra`をリリースしました。
-- 従来型の解析モデルパッケージ`ja-ginza`のpiplelineに`morphologizer`を追加し、UD品詞解析精度を向上しました。
-- `Token.lemma_`にSudachiPyのnormalized_formを強制的にセットするオプション(`ginza -n`)を追加しました。
+- `spaCy v3.2`と`Sudachi.rs(SudachiPy v0.6.2)`に対応した`GiNZA v5.1`をリリース
+  - バッチ解析処理をGPU環境で50〜60%・CPU環境で10〜40%高速化
+  - ginzaコマンドの並列実行オプション(`ginza -p {n_process}`および`ginzame`)の処理効率を向上
+- ginzaコマンドで日本語以外を含む全てのspaCyモデルが利用可能に
+  - `ginza -m en_core_web_md` の形でモデル名を指定することで[CoNLL-U](https://universaldependencies.org/format.html#syntactic-annotation)出力ツールとして利用可能
+- [ginzaコマンドの解説ページ](https://megagonlabs.github.io/ginza/command_line_tool.html)の記述を拡充
+  - `ginza`コマンドで使用するGPUのgpu_idを`ginza -g 1`の形で指定可能に
 
 ***GiNZAをアップグレードする際は下記の互換性情報を確認してください。***
 
-## GiNZA v5.0 互換性情報
-- v4以前の`ginza`および`ja-ginza`パケージはGiNZA v5で使用できません（旧バージョン向けパッケージは事前にアンインストールが必要です）
-  - `pip uninstall ginza ja-ginza`
-- transformersモデルの追加に伴いGiNZA v5インストール時は`ginza`パッケージとともに解析モデルパッケージを明示的に指定する必要があります
+## GiNZA v5.1 互換性情報
+- `ginza --require_gpu`および`ginza -g`オプションが引数にgpu_idを取るようになりました
+  - gpu_idに-1を指定(デフォルト)するとCPUのみを使用します
+- v5.0以前の`ja_ginza`および`ja_ginza_electra`パケージはGiNZA v5.1で使用できません（旧バージョン向けパッケージは事前にアンインストールが必要です）
+  - `pip uninstall ginza ; pip uninstall ja_ginza ; pip uninstall ja_ginza_electra`
+- transformersモデルの追加に伴いGiNZA v5.1インストール時は`ginza`パッケージとともに解析モデルパッケージを明示的に指定する必要があります
   - 解析精度重視モデル (メモリ容量16GB以上を推奨)
-    - `pip install -U ginza ja-ginza-electra`
+    - `pip install -U ginza ja_ginza_electra`
   - 実行速度重視モデル
-    - `pip install -U ginza ja-ginza`
-- `CompoundSplitter`および`BunsetuRecognizer`の名称を`compound_splitter`および`bunsetu_recognizer`に変更しました
-- 併せてspaCy v3の[Backwards Incompatibilities](https://spacy.io/usage/v3#incompat)も確認してください
+    - `pip install -U ginza ja_ginza`
+- トークンの活用/読み/正規形の保存先がTokenクラスのフィールドに変更されました。
+  - `doc.user_data[“reading_forms”][token.i]` -> `token.morph.get(“Reading”)`
+  - `doc.user_data[“inflections”][token.i]` -> `token.morph.get(“Inflection”)`
+  - `force_using_normalized_form_as_lemma(True)` -> `token.norm_`
+- 併せて[GiNZA v5.0.0 重要な変更](#ginza-500)についても確認してください
 
 ## GiNZA v5 新機能
 
@@ -62,17 +70,27 @@ GiNZAをインストールする前に予めPython実行環境を構築してく
 
 ※Transformersモデルの実行には16GB以上のメモリ容量が必要です。メモリ容量が不足する場合は後述の従来型モデルをお試しください。
 
-旧バージョンのGiNZAをインストールしている場合は次のコマンドでアンインストールを実行します。
+旧バージョンのGiNZAをインストールしている場合は次のコマンドでアンインストールします。
 ```console
-$ pip uninstall ginza ja-ginza
+$ pip uninstall ginza ja_ginza_electra
+```
+
+旧バージョンの`ja_ginza`をインストールしている場合は次のコマンドでアンインストールします。
+```console
+$ pip uninstall ja_ginza
+```
+
+旧バージョンの`j_ginza_electra`をインストールしている場合は次のコマンドでアンインストールします。
+```console
+$ pip uninstall ja_ginza_electra
 ```
 
 次のコマンドを実行して最新のGiNZAおよびTransformersモデルをインストールします。
 ```console
-$ pip install -U ginza ja-ginza-electra
+$ pip install -U ginza ja_ginza_electra
 ```
 
-上記コマンドでインストールされる`ja-ginza-electra`パッケージには大容量モデルファイルは含まれていません。大容量モデルファイルは初回実行時に自動的にダウンロードされて、以降の実行時にはローカルにキャッシュされたファイルが使用されます。
+上記コマンドでインストールされる`ja_ginza_electra`パッケージには大容量モデルファイルは含まれていません。大容量モデルファイルは初回実行時に自動的にダウンロードされて、以降の実行時にはローカルにキャッシュされたファイルが使用されます。
 
 大容量モデルファイルを含めたインストールを行うには、次のようにGitHubのリリースアーカイブを指定します。
 ```console
@@ -86,16 +104,18 @@ CUDA 11.0を使用する場合:
 pip install -U "spacy[cuda110]"
 ```
 
+あわせてpytorchもCUDAと整合したバージョンをインストールする必要があります。
+
 #### 2. GiNZA + 従来型モデル
 
-旧バージョンのGiNZAをインストールしている場合は次のコマンドでアンインストールを実行します。
+旧バージョンのGiNZAおよび`ja_ginza`をインストールしている場合は次のコマンドでアンインストールを実行します。
 ```console
-$ pip uninstall ginza ja-ginza
+$ pip uninstall ginza _ginza
 ```
 
 次のコマンドを実行して最新のGiNZAと従来型モデルをインストールします。
 ```console
-$ pip install -U ginza ja-ginza
+$ pip install -U ginza ja_ginza
 ```
 
 ### ginzaコマンドによる解析処理の実行
@@ -121,6 +141,11 @@ $ ginza
 # ginza -m ja_ginza
 ```
 
+また、spaCyが提供している[様々な言語の解析モデル](https://spacy.io/usage/models)の名称を`-m`オプションに指定することで、そのモデルのダウンロードと解析をまとめて実行することができます。
+```console
+# ginza -m en_core_web_trf
+```
+
 ### Pythonコードによる解析処理の実行
 
 次のコードは、Transformersモデルによる依存構造解析結果を文単位で出力します。
@@ -131,7 +156,18 @@ nlp = spacy.load('ja_ginza_electra')
 doc = nlp('銀座でランチをご一緒しましょう。')
 for sent in doc.sents:
     for token in sent:
-        print(token.i, token.orth_, token.lemma_, token.pos_, token.tag_, token.dep_, token.head.i)
+        print(
+          token.i,
+          token.orth_,
+          token.lemma_,
+          token.norm_
+          token.morph.get(“Reading”),
+          token.pos_,
+          token.morph.get(“Inflection”),
+          token.tag_,
+          token.dep_,
+          token.head.i,
+        )
     print('EOS')
 ```
 
@@ -204,7 +240,7 @@ GiNZA v5の固有表現抽出モデルは
 GiNZA v5の固有表現抽出モデルは国立国語研究所とMegagon Labsの共同研究成果です。
 
 ### mC4
-GiNZA v5 Transformersモデル(ja-ginza-electra)は、[mC4](https://huggingface.co/datasets/mc4)から抽出した日本語20億文以上を用いて事前学習した[transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator)を使用しています。
+GiNZA v5 Transformersモデル(ja_ginza_electra)は、[mC4](https://huggingface.co/datasets/mc4)から抽出した日本語20億文以上を用いて事前学習した[transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator)を使用しています。
 mC4はODC-BYライセンスの規約に基づいて事前学習データとして利用しています。
 
 Contains information from mC4 which is made available under the ODC Attribution License.
@@ -225,6 +261,26 @@ Contains information from mC4 which is made available under the ODC Attribution
 
 ### version 5.x
 
+#### ginza-5.1.0
+- 2021-12-10, Euclase
+- 重要な変更
+  - spaCy v3.2 および Sudachi.rs(SudachiPy v0.6.2) に対応
+  - トークンの活用・読み・正規形の保存先をTokenクラスのフィールドに変更 #208 #209
+    - `doc.user_data[“reading_forms”][token.i]` -> `token.morph.get(“Reading”)`
+    - `doc.user_data[“inflections”][token.i]` -> `token.morph.get(“Inflection”)`
+    - `force_using_normalized_form_as_lemma(True)` -> `token.norm_`
+  - ginzaコマンドで日本語以外を含む全てのspaCyモデルが利用可能に #217
+    - `ginza -m en_core_web_md` の形でモデル名を指定することでモデルのダウンロードと解析をまとめて実行 #219
+  - `ginza --require_gpu`および`ginza -g`オプションがgpu_idを引数を取る形に変更
+    - -1を指定(デフォルト)するとCPUのみを使用
+  - ginza -f json で -c オプションの指定に関わらず#で始まるはもすべて解析対象とする #215
+- Improvements
+  - バッチ解析処理をGPU環境で50〜60%・CPU環境で10〜40%高速化
+  - ginzaコマンドの並列実行オプション(`ginza -p {n_process}`および`ginzame`)の処理効率を向上 #204
+  - [ginzaコマンドの解説ページ](https://megagonlabs.github.io/ginza/command_line_tool.html)の記述を拡充 #201
+  - add tests #198 #210 #214
+  - add benchmark #207 #220
+
 #### ginza-5.0.3
 - 2021-10-15
 - Bug fix
diff --git a/ginza/__init__.py b/ginza/__init__.py
index 43f101e..3738242 100644
--- a/ginza/__init__.py
+++ b/ginza/__init__.py
@@ -109,8 +109,9 @@ def force_using_normalized_form_as_lemma(force: bool):
 
 
 def set_split_mode(nlp: Language, mode: str):
-    splitter = nlp.get_pipe("compound_splitter")
-    splitter.split_mode = mode
+    if nlp.has_pipe("compound_splitter"):
+        splitter = nlp.get_pipe("compound_splitter")
+        splitter.split_mode = mode
 
 
 # token field getters
@@ -219,15 +220,22 @@ def ent_label_ontonotes(token: Token) -> str:
 
 # token field getters for Doc.user_data
 
-def reading_form(token: Token, use_orth_if_none=True) -> str:
-    reading = token.doc.user_data["reading_forms"][token.i]
-    if not reading and use_orth_if_none:
-        reading = token.orth_
-    return reading
+def reading_form(token: Token, use_orth_if_none: bool) -> str:
+    reading = token.morph.get("Reading")
+    if reading:
+        return reading[0]
+    elif use_orth_if_none:
+        return token.orth_
+    else:
+        return None
 
 
 def inflection(token: Token) -> str:
-    return token.doc.user_data["inflections"][token.i]
+    inf = token.morph.get("Inflection")
+    if inf:
+        return inf[0].replace(";", ",")
+    else:
+        return ""
 
 
 # bunsetu related field getters for Doc.user_data
@@ -447,7 +455,7 @@ def _sub_tokens(
                 token.tag_,
                 inflection(token),
                 token.lemma_,
-                reading_form(token),
+                reading_form(token, True),
                 None,
             )
         ]
diff --git a/ginza/analyzer.py b/ginza/analyzer.py
new file mode 100644
index 0000000..5e1f7c6
--- /dev/null
+++ b/ginza/analyzer.py
@@ -0,0 +1,321 @@
+# coding: utf8
+import sys
+from typing import Iterable, Optional
+
+import thinc
+
+import spacy
+from spacy.tokens import Doc, Span
+from spacy.language import Language
+from spacy.lang.ja import Japanese
+
+from . import set_split_mode, inflection, reading_form, ent_label_ene, ent_label_ontonotes, bunsetu_bi_label, bunsetu_position_type
+from .bunsetu_recognizer import bunsetu_available, bunsetu_head_list, bunsetu_phrase_span
+
+
+def try_sudachi_import(split_mode: str):
+    """SudachiPy is required for Japanese support, so check for it.
+    It it's not available blow up and explain how to fix it.
+    split_mode should be one of these values: "A", "B", "C", None->"A"."""
+    try:
+        from sudachipy import dictionary, tokenizer
+
+        split_mode = {
+            None: tokenizer.Tokenizer.SplitMode.A,
+            "A": tokenizer.Tokenizer.SplitMode.A,
+            "B": tokenizer.Tokenizer.SplitMode.B,
+            "C": tokenizer.Tokenizer.SplitMode.C,
+        }[split_mode]
+        tok = dictionary.Dictionary().create(mode=split_mode)
+        return tok
+    except ImportError:
+        raise ImportError(
+            "Japanese support requires SudachiPy and SudachiDict-core "
+            "(https://github.com/WorksApplications/SudachiPy). "
+            "Install with `pip install sudachipy sudachidict_core` or "
+            "install spaCy with `pip install spacy[ja]`."
+        ) from None
+
+
+class Analyzer:
+    def __init__(
+        self,
+        model_name_or_path: str,
+        split_mode: str,
+        hash_comment: str,
+        output_format: str,
+        require_gpu: int,
+        disable_sentencizer: bool,
+        use_normalized_form: bool,
+    ) -> None:
+        self.model_name_or_path = model_name_or_path
+        self.split_mode = split_mode
+        self.hash_comment = hash_comment
+        self.output_format = output_format
+        self.require_gpu = require_gpu
+        self.disable_sentencizer = disable_sentencizer
+        self.use_normalized_form = use_normalized_form
+        self.nlp: Optional[Language] = None
+
+    def set_nlp(self) -> None:
+        if self.nlp:
+            return
+
+        if self.require_gpu >= 0:
+            thinc.api.require_gpu(self.require_gpu)
+
+        if self.output_format in ["2", "mecab"]:
+            nlp = try_sudachi_import(self.split_mode)
+        else:
+            # Work-around for pickle error. Need to share model data.
+            if self.model_name_or_path:
+                nlp = spacy.load(self.model_name_or_path)
+            else:
+                try:
+                    nlp = spacy.load("ja_ginza_electra")
+                except IOError as e:
+                    try:
+                        nlp = spacy.load("ja_ginza")
+                    except IOError as e:
+                        raise OSError("E050", 'You need to install "ja-ginza" or "ja-ginza-electra" by executing `pip install ja-ginza`.')
+
+            if self.disable_sentencizer:
+                nlp.add_pipe("disable_sentencizer", before="parser")
+
+            if self.split_mode:
+                set_split_mode(nlp, self.split_mode)
+
+        self.nlp = nlp
+        self.use_orth_if_reading_is_none = isinstance(self.nlp, Japanese)
+
+    def analyze_batch(self, lines: Iterable[str]) -> str:
+        self.set_nlp()
+        if self.output_format in ["2", "mecab"]:
+            return "".join(self.analyze_line(line) for line in lines)
+
+        if self.hash_comment == "print":
+            batch = list(self.nlp.pipe(line.rstrip("\n") for line in lines if not line.startswith("#")))
+            docs = []
+            index = 0
+            for line in lines:
+                if line.startswith("#"):
+                    docs.append(line)
+                else:
+                    docs.append(batch[index])
+                    index += 1
+        else:
+            lines = [line.rstrip("\n") for line in lines if self.hash_comment != "skip" or not line.startswith("#")]
+            docs = self.nlp.pipe(lines)
+
+        if self.output_format in ["3", "json"]:
+            sep = ",\n"
+        else:
+            sep = ""
+        return sep.join(format_doc(doc, self.output_format, self.use_normalized_form, self.use_orth_if_reading_is_none) if isinstance(doc, Doc) else doc for doc in docs)
+
+    def analyze_line(self, input_line: str) -> str:
+        line = input_line.rstrip("\n")
+        if line.startswith("#"):
+            if self.hash_comment == "print":
+                return input_line
+            elif self.hash_comment == "skip":
+                return ""
+        if line == "":
+            return "\n"
+        if self.output_format in ["2", "mecab"]:
+            doc = self.nlp.tokenize(line)
+        else:
+            doc = self.nlp(line)
+        return format_doc(doc, self.output_format, self.use_normalized_form, self.use_orth_if_reading_is_none)
+
+
+def format_doc(
+   doc: Doc, output_format: str, use_normalized_form: bool, use_orth_if_reading_is_none: bool,
+) -> str:
+    if output_format in ["0", "conllu"]:
+        return "".join(format_conllu(sent, use_normalized_form, use_orth_if_reading_is_none) for sent in doc.sents)
+    elif output_format in ["1", "cabocha"]:
+        return "".join(format_cabocha(sent, use_normalized_form) for sent in doc.sents)
+    elif output_format in ["2", "mecab"]:
+        return "".join(format_mecab(doc, use_normalized_form))
+    elif output_format in ["3", "json"]:
+        return ",\n".join(format_json(sent) for sent in doc.sents)
+    else:
+        raise Exception(output_format + " is not supported")
+
+
+def format_json(sent: Span) -> str:
+    token_lines = ",\n".join(
+        f"""       {{"id":{
+            token.i - sent.start + 1
+        },"orth":"{
+            token.orth_
+        }","tag":"{
+            token.tag_
+        }","pos":"{
+            token.pos_
+        }","lemma":"{
+            token.lemma_
+        }","norm":"{
+            token.norm_
+        }","head":{
+            token.head.i - token.i
+        },"dep":"{
+            token.dep_
+        }","ner":"{
+            token.ent_iob_
+        }{
+            "-" + token.ent_type_ if token.ent_type_ else ""
+        }"{
+            ',"whitespacce":"' + token.whitespace_ + '"' if token.whitespace_ else ""
+        }}}""" for token in sent
+    )
+    return f""" {{
+  "paragraphs": [
+   {{
+    "raw": "{sent.text}",
+    "sentences": [
+     {{
+      "tokens": [
+{token_lines}
+      ]
+     }}
+    ]
+   }}
+  ]
+ }}"""
+
+
+def format_conllu(sent: Span, use_normalized_form, use_orth_if_reading_is_none, print_origin=True) -> str:
+    np_labels = [""] * len(sent)
+    use_bunsetu = bunsetu_available(sent)
+    if use_bunsetu:
+        for head_i in bunsetu_head_list(sent):
+            bunsetu_head_token = sent[head_i]
+            phrase = bunsetu_phrase_span(bunsetu_head_token)
+            if phrase.label_ == "NP":
+                for idx in range(phrase.start - sent.start, phrase.end - sent.start):
+                    np_labels[idx] = "NP_B" if idx == phrase.start else "NP_I"
+    token_lines = "".join(conllu_token_line(sent, token, np_label, use_bunsetu, use_normalized_form, use_orth_if_reading_is_none) for token, np_label in zip(sent, np_labels))
+    if print_origin:
+        return f"# text = {sent.text}\n{token_lines}\n"
+    else:
+        return f"{token_lines}\n"
+
+
+def conllu_token_line(sent, token, np_label, use_bunsetu, use_normalized_form, use_orth_if_reading_is_none) -> str:
+    bunsetu_bi = bunsetu_bi_label(token) if use_bunsetu else None
+    position_type = bunsetu_position_type(token) if use_bunsetu else None
+    inf = inflection(token)
+    reading = reading_form(token, use_orth_if_reading_is_none)
+    ne = ent_label_ontonotes(token)
+    ene = ent_label_ene(token)
+    misc = "|".join(
+        filter(
+            lambda s: s,
+            (
+                "SpaceAfter=Yes" if token.whitespace_ else "SpaceAfter=No",
+                "" if not bunsetu_bi else "BunsetuBILabel={}".format(bunsetu_bi),
+                "" if not position_type else "BunsetuPositionType={}".format(position_type),
+                np_label,
+                "" if not inf else "Inf={}".format(inf),
+                "" if not reading else "Reading={}".format(reading.replace("|", "\\|").replace("\\", "\\\\")),
+                "" if not ne or ne == "O" else "NE={}".format(ne),
+                "" if not ene or ene == "O" else "ENE={}".format(ene),
+            )
+        )
+    )
+
+    return "\t".join(
+        [
+            str(token.i - sent.start + 1),
+            token.orth_,
+            token.norm_ if use_normalized_form else token.lemma_,
+            token.pos_,
+            token.tag_.replace(",*", "").replace(",", "-"),
+            "NumType=Card" if token.pos_ == "NUM" else "_",
+            "0" if token.head.i == token.i else str(token.head.i - sent.start + 1),
+            token.dep_.lower() if token.dep_ else "_",
+            "_",
+            misc if misc else "_",
+        ]
+    ) + "\n"
+
+
+def format_cabocha(sent: Span, use_normalized_form) -> str:
+    bunsetu_index_list = {}
+    bunsetu_index = -1
+    for token in sent:
+        if bunsetu_bi_label(token) == "B":
+            bunsetu_index += 1
+        bunsetu_index_list[token.i] = bunsetu_index
+
+    lines = []
+    for token in sent:
+        if bunsetu_bi_label(token) == "B":
+            lines.append(cabocha_bunsetu_line(sent, bunsetu_index_list, token))
+        lines.append(cabocha_token_line(token, use_normalized_form))
+    lines.append("EOS\n\n")
+    return "".join(lines)
+
+
+def cabocha_bunsetu_line(sent: Span, bunsetu_index_list, token) -> str:
+    bunsetu_head_index = None
+    bunsetu_dep_index = None
+    bunsetu_func_index = None
+    dep_type = "D"
+    for t in token.doc[token.i : sent.end]:
+        if bunsetu_index_list[t.i] != bunsetu_index_list[token.i]:
+            if bunsetu_func_index is None:
+                bunsetu_func_index = t.i - token.i
+            break
+        tbi = bunsetu_index_list[t.head.i]
+        if bunsetu_index_list[t.i] != tbi:
+            bunsetu_head_index = t.i - token.i
+            bunsetu_dep_index = tbi
+        if bunsetu_func_index is None and bunsetu_position_type(t) in {"FUNC", "SYN_HEAD"}:
+            bunsetu_func_index = t.i - token.i
+    else:
+        if bunsetu_func_index is None:
+            bunsetu_func_index = len(sent) - token.i
+    if bunsetu_head_index is None:
+        bunsetu_head_index = 0
+    if bunsetu_dep_index is None:
+        bunsetu_dep_index = -1
+    return "* {} {}{} {}/{} 0.000000\n".format(
+        bunsetu_index_list[token.i],
+        bunsetu_dep_index,
+        dep_type,
+        bunsetu_head_index,
+        bunsetu_func_index,
+    )
+
+
+def cabocha_token_line(token, use_normalized_form) -> str:
+    part_of_speech = token.tag_.replace("-", ",")
+    inf = inflection(token)
+    part_of_speech += ",*" * (3 - part_of_speech.count(",")) + "," + (inf if inf else "*,*")
+    reading = reading_form(token, True)
+    return "{}\t{},{},{},{}\t{}\n".format(
+        token.orth_,
+        part_of_speech,
+        token.norm_ if use_normalized_form else token.lemma_,
+        reading if reading else token.orth_,
+        "*",
+        "O" if token.ent_iob_ == "O" else "{}-{}".format(token.ent_iob_, token.ent_type_),
+    )
+
+
+def format_mecab(sudachipy_tokens, use_normalized_form) -> str:
+    return "".join(mecab_token_line(t, use_normalized_form) for t in sudachipy_tokens) + "EOS\n\n"
+
+
+def mecab_token_line(token, use_normalized_form) -> str:
+    reading = token.reading_form()
+    return "{}\t{},{},{},{}\n".format(
+        token.surface(),
+        ",".join(token.part_of_speech()),
+        token.normalized_form() if use_normalized_form else token.dictionary_form(),
+        reading if reading else token.surface(),
+        "*",
+    )
diff --git a/ginza/command_line.py b/ginza/command_line.py
index 290586e..01c25e7 100644
--- a/ginza/command_line.py
+++ b/ginza/command_line.py
@@ -1,434 +1,308 @@
 # coding: utf8
-import json
-from multiprocessing import Pool, cpu_count
+from multiprocessing import Process, Queue, Event, cpu_count
 from pathlib import Path
+import queue
+import re
 import sys
+import traceback
+from typing import Generator, Iterable, Optional, List
 
 import plac
-import spacy
-from spacy.tokens import Span
+from .analyzer import Analyzer
 
-from spacy.lang.ja import Japanese, JapaneseTokenizer
+MINI_BATCH_SIZE = 100
+GINZA_MODEL_PATTERN = re.compile(r"^(ja_ginza|ja_ginza_electra)$")
+SPACY_MODEL_PATTERN = re.compile(r"^[a-z]{2}[-_].+[-_].+(sm|md|lg|trf)$")
 
-from . import set_split_mode, inflection, reading_form, ent_label_ene, ent_label_ontonotes,\
-    bunsetu_bi_label, bunsetu_position_type, force_using_normalized_form_as_lemma, make_disable_sentencizer
-from .bunsetu_recognizer import bunsetu_available, bunsetu_head_list, bunsetu_phrase_span
 
-MINI_BATCH_SIZE = 100
+class _OutputWrapper:
+    def __init__(self, output_path, output_format):
+        self.output = None
+        self.output_path = output_path
+        self.output_format = output_format
+        self.output_json_opened = False
+
+    @property
+    def is_json(self):
+        return self.output_format in ["3", "json"]
+
+    def open(self):
+        if self.output_path:
+            self.output = open(self.output_path, "w")
+        else:
+            self.output = sys.stdout
+
+    def close(self):
+        if self.is_json and self.output_json_opened:
+            print("\n]", file=self.output)
+            self.output_json_opened = False
+        if self.output_path:
+            self.output.close()
+        else:
+            pass
+
+    def write(self, result: str):
+        if self.is_json:
+            if not self.output_json_opened:
+                print("[", file=self.output)
+                self.output_json_opened = True
+            else:
+                print(",", file=self.output)
+        print(result, end="", file=self.output)
 
 
 def run(
-        model_path=None,
-        ensure_model=None,
-        split_mode=False,
-        hash_comment="print",
-        output_path=None,
-        output_format="0",
-        require_gpu=False,
-        disable_sentencizer=False,
-        use_normalized_form=False,
-        parallel=1,
-        files=None,
+    model_path: Optional[str] = None,
+    ensure_model: Optional[str] = None,
+    split_mode: Optional[str] = None,
+    hash_comment: str = "print",
+    output_path: Optional[Path] = None,
+    output_format: str = "0",
+    require_gpu: int = -1,
+    disable_sentencizer: bool = False,
+    use_normalized_form: bool = False,
+    parallel_level: int = 1,
+    files: List[str] = None,
 ):
-    if require_gpu:
-        print("GPU enabled", file=sys.stderr)
-    if use_normalized_form:
-        print("overriding Token.lemma_ by normalized_form of SudachiPy", file=sys.stderr)
-        force_using_normalized_form_as_lemma(True)
+    if output_format in ["3", "json"] and hash_comment != "analyze":
+        print(
+            f'hash_comment="{hash_comment}" not permitted for JSON output. Forced to use hash_comment="analyze".',
+            file=sys.stderr
+        )
+
+    assert parallel_level == 1 or require_gpu == -1, "require_gpu not allowed for multi-processing. https://github.com/explosion/spaCy/issues/5507"
+
+    if parallel_level <= 0:
+        level = max(1, cpu_count() + parallel_level)
+        if output_format in [2, "mecab"]:
+            if require_gpu >= 0:
+                print("GPU not used for mecab mode", file=sys.stderr)
+                require_gpu = False
+        elif parallel_level <= 0:
+            if require_gpu >= 0:
+                if level < 4:
+                    print(f"GPU #{require_gpu} enabled: parallel_level' set to {level}", end="", file=sys.stderr)
+                else:
+                    print(f"GPU #{require_gpu} enabled: parallel_level' set to {level} but seems it's too much", end="", file=sys.stderr)
+            else:
+                print(f"'parallel_level' set to {level}", file=sys.stderr)
+        elif require_gpu:
+            print(f"GPU #{require_gpu} enabled", file=sys.stderr)
+        parallel_level = level
+
     assert model_path is None or ensure_model is None
+    if ensure_model:
+        ensure_model = ensure_model.replace("-", "_")
+        try:
+            from importlib import import_module
+            import_module(ensure_model)
+        except ModuleNotFoundError:
+            if GINZA_MODEL_PATTERN.match(ensure_model):
+                print("Installing", ensure_model, file=sys.stderr)
+                import pip
+                pip.main(["install", ensure_model])
+                print("Successfully installed", ensure_model, file=sys.stderr)
+            elif SPACY_MODEL_PATTERN.match(ensure_model):
+                print("Installing", ensure_model, file=sys.stderr)
+                from spacy.cli.download import download
+                download(ensure_model)
+                print("Successfully installed", ensure_model, file=sys.stderr)
+            else:
+                raise OSError("E050", f'You need to install "{ensure_model}" before executing ginza.')
+        model_name_or_path = ensure_model
+    else:
+        model_name_or_path = model_path
 
     analyzer = Analyzer(
-        model_path,
-        ensure_model,
+        model_name_or_path,
         split_mode,
         hash_comment,
         output_format,
         require_gpu,
         disable_sentencizer,
+        use_normalized_form,
     )
 
-    if parallel <= 0:
-        parallel = max(1, cpu_count() + parallel)
-
-    pool = None
-
-    if output_path:
-        output = open(str(output_path), "w")
-    else:
-        output = sys.stdout
-
-    output_json = output_format in ["3", "json"]
-    output_json_opened = False
-
-    def output_json_open():
-        nonlocal output_json_opened
-        if output_json and not output_json_opened:
-            print("[", file=output)
-            output_json_opened = True
-        elif output_json:
-            print(" ,", file=output)
-
+    output = _OutputWrapper(output_path, output_format)
+    output.open()
     try:
-        if not files:
-            if sys.stdin.isatty():
-                parallel = 1
-            else:
+        if not files and sys.stdin.isatty():
+            _analyze_tty(analyzer, output)
+        else:
+            if not files:
                 files = [0]
+            if parallel_level == 1:
+                _analyze_single(analyzer, output, files)
+            else:
+                _analyze_parallel(analyzer, output, files, parallel_level)
+    finally:
+        output.close()
 
-        if not files:
-            analyzer.set_nlp()
-            while True:
-                line = input()
-                for sent in analyzer.analyze_line(line):
-                    for ol in sent:
-                        output_json_open()
-                        print(ol, file=output)
-        elif parallel == 1:
-            analyzer.set_nlp()
-            for path in files:
-                with open(path, "r") as f:
-                    for line in f:
-                        for sent in analyzer.analyze_line(line):
-                            for ol in sent:
-                                output_json_open()
-                                print(ol, file=output)
-        else:
-            buffer = []
-            for file_idx, path in enumerate(files):
-                with open(path, "r") as f:
-                    while True:
-                        eof, buffer = fill_buffer(f, MINI_BATCH_SIZE * parallel, buffer)
-                        if eof and (file_idx + 1 < len(files) or len(buffer) == 0):
-                            break  # continue to next file
-                        if not pool:
-                            if len(buffer) <= MINI_BATCH_SIZE:  # enough for single process
-                                analyzer.set_nlp()
-                                for line in buffer:
-                                    for sent in analyzer.analyze_line(line):
-                                        for ol in sent:
-                                            output_json_open()
-                                            print(ol, file=output)
-                                break  # continue to next file
-                            parallel = (len(buffer) - 1) // MINI_BATCH_SIZE + 1
-                            pool = Pool(parallel)
-
-                        mini_batch_size = (len(buffer) - 1) // parallel + 1
-                        mini_batches = [
-                            buffer[idx * mini_batch_size:(idx + 1) * mini_batch_size] for idx in range(parallel)
-                        ]
-                        for mini_batch_result in pool.map(analyzer.analyze_lines_mp, mini_batches):
-                            for sents in mini_batch_result:
-                                for lines in sents:
-                                    for ol in lines:
-                                        output_json_open()
-                                        print(ol, file=output)
-
-                        buffer.clear()  # process remaining part of current file
 
+def _analyze_tty(analyzer: Analyzer, output: _OutputWrapper) -> None:
+    try:
+        analyzer.set_nlp()
+        while True:
+            line = input()
+            output.write(analyzer.analyze_line(line))
     except EOFError:
         pass
     except KeyboardInterrupt:
         pass
-    finally:
-        try:
-            if pool:
-                pool.close()
-        finally:
-            if output_json and output_json_opened:
-                print("]", file=output)
-            output.close()
-
-
-def fill_buffer(f, batch_size, buffer=None):
-    if buffer is None:
-        buffer = []
-
-    for line in f:
-        buffer.append(line)
-        if len(buffer) == batch_size:
-            return False, buffer
-    return True, buffer
-
-
-class Analyzer:
-    def __init__(
-            self,
-            model_path,
-            ensure_model,
-            split_mode,
-            hash_comment,
-            output_format,
-            require_gpu,
-            disable_sentencizer,
-    ):
-        self.model_path = model_path
-        self.ensure_model = ensure_model
-        self.split_mode = split_mode
-        self.hash_comment = hash_comment
-        self.output_format = output_format
-        self.require_gpu = require_gpu
-        self.disable_sentencizer = disable_sentencizer
-        self.nlp = None
 
-    def set_nlp(self):
-        if self.nlp:
-            return
 
-        if self.require_gpu:
-            spacy.require_gpu()
+def _analyze_single(analyzer: Analyzer, output: _OutputWrapper, files: Iterable[str]) -> None:
+    try:
+        analyzer.set_nlp()
+        for path in files:
+            with open(path, "r") as f:
+                for line in f:
+                    output.write(analyzer.analyze_line(line))
+    except KeyboardInterrupt:
+        pass
 
-        if self.output_format in ["2", "mecab"]:
-            nlp = JapaneseTokenizer(nlp=Japanese(), split_mode=self.split_mode).tokenizer
-        else:
-            # Work-around for pickle error. Need to share model data.
-            if self.model_path:
-                nlp = spacy.load(self.model_path)
-            elif self.ensure_model:
-                    nlp = spacy.load(self.ensure_model.replace("-", "_"))
-            else:
-                try:
-                    nlp = spacy.load("ja_ginza_electra")
-                except IOError as e:
-                    try:
-                        nlp = spacy.load("ja_ginza")
-                    except IOError as e:
-                        print('Could not find the model. You need to install "ja-ginza-electra" or "ja-ginza" by executing pip like `pip install ja-ginza-electra`.', file=sys.stderr)
-                        raise e
-
-            if self.disable_sentencizer:
-                nlp.add_pipe("disable_sentencizer", before="parser")
-
-            if self.split_mode:
-                set_split_mode(nlp, self.split_mode)
-
-        self.nlp = nlp
-
-    def analyze_lines_mp(self, lines):
-        self.set_nlp()
-        return tuple(self.analyze_line(line) for line in lines)
-
-    def analyze_line(self, line):
-        return analyze(self.nlp, self.hash_comment, self.output_format, line)
-
-
-def analyze(nlp, hash_comment, output_format, line):
-    line = line.rstrip("\n")
-    if line.startswith("#"):
-        if hash_comment == "print":
-            return line,
-        elif hash_comment == "skip":
-            return (),
-    if line == "":
-        return "",
-    if output_format in ["0", "conllu"]:
-        doc = nlp(line)
-        return [analyze_conllu(sent) for sent in doc.sents]
-    elif output_format in ["1", "cabocha"]:
-        doc = nlp(line)
-        return [analyze_cabocha(sent) for sent in doc.sents]
-    elif output_format in ["2", "mecab"]:
-        doc = nlp.tokenize(line)
-        return [analyze_mecab(doc)]
-    elif output_format in ["3", "json"]:
-        doc = nlp(line)
-        return [analyze_json(sent) for sent in doc.sents]
-    else:
-        raise Exception(output_format + " is not supported")
-
-
-def analyze_json(sent: Span):
-    tokens = []
-    for token in sent:
-        t = {
-            "id": token.i - sent.start + 1,
-            "orth": token.orth_,
-            "tag": token.tag_,
-            "pos": token.pos_,
-            "lemma": token.lemma_,
-            "head": token.head.i - token.i,
-            "dep": token.dep_,
-            "ner": "{}-{}".format(token.ent_iob_, token.ent_type_) if token.ent_type_ else token.ent_iob_,
-        }
-        if token.whitespace_:
-            t["whitespace"] = token.whitespace_
-        tokens.append("       " + json.dumps(t, ensure_ascii=False))
-    tokens = ",\n".join(tokens)
-
-    yield """ {{
-  "paragraphs": [
-   {{
-    "raw": "{}",
-    "sentences": [
-     {{
-      "tokens": [
-{}
-      ]
-     }}
-    ]
-   }}
-  ]
- }}""".format(
-        sent.text,
-        tokens,
-    )
 
+def _analyze_parallel(analyzer: Analyzer, output: _OutputWrapper, files: Iterable[str], parallel_level: int) -> None:
+    try:
+        in_queue = Queue(maxsize=parallel_level * 2)
+        out_queue = Queue()
 
-def analyze_conllu(sent: Span, print_origin=True):
-    if print_origin:
-        yield "# text = {}".format(sent.text)
-    np_labels = [""] * len(sent)
-    use_bunsetu = bunsetu_available(sent)
-    if use_bunsetu:
-        for head_i in bunsetu_head_list(sent):
-            bunsetu_head_token = sent[head_i]
-            phrase = bunsetu_phrase_span(bunsetu_head_token)
-            if phrase.label_ == "NP":
-                for idx in range(phrase.start - sent.start, phrase.end - sent.start):
-                    np_labels[idx] = "NP_B" if idx == phrase.start else "NP_I"
-    for token, np_label in zip(sent, np_labels):
-        yield conllu_token_line(sent, token, np_label, use_bunsetu)
-    yield ""
-
-
-def conllu_token_line(sent, token, np_label, use_bunsetu):
-    bunsetu_bi = bunsetu_bi_label(token) if use_bunsetu else None
-    position_type = bunsetu_position_type(token) if use_bunsetu else None
-    inf = inflection(token)
-    reading = reading_form(token)
-    ne = ent_label_ontonotes(token)
-    ene = ent_label_ene(token)
-    misc = "|".join(filter(lambda s: s, (
-        "SpaceAfter=Yes" if token.whitespace_ else "SpaceAfter=No",
-        "" if not bunsetu_bi else "BunsetuBILabel={}".format(bunsetu_bi),
-        "" if not position_type else "BunsetuPositionType={}".format(position_type),
-        np_label,
-        "" if not inf else "Inf={}".format(inf),
-        "" if not reading else "Reading={}".format(reading.replace("|", "\\|").replace("\\", "\\\\")),
-        "" if not ne or ne == "O" else "NE={}".format(ne),
-        "" if not ene or ene == "O" else "ENE={}".format(ene),
-    )))
-
-    return "\t".join([
-        str(token.i - sent.start + 1),
-        token.orth_,
-        token.lemma_,
-        token.pos_,
-        token.tag_.replace(",*", "").replace(",", "-"),
-        "NumType=Card" if token.pos_ == "NUM" else "_",
-        "0" if token.head.i == token.i else str(token.head.i - sent.start + 1),
-        token.dep_.lower() if token.dep_ else "_",
-        "_",
-        misc if misc else "_",
-    ])
-
-
-def analyze_cabocha(sent: Span):
-    bunsetu_index_list = {}
-    bunsetu_index = -1
-    for token in sent:
-        if bunsetu_bi_label(token) == "B":
-            bunsetu_index += 1
-        bunsetu_index_list[token.i] = bunsetu_index
-
-    lines = []
-    for token in sent:
-        if bunsetu_bi_label(token) == "B":
-            lines.append(cabocha_bunsetu_line(sent, bunsetu_index_list, token))
-        lines.append(cabocha_token_line(token))
-    lines.append("EOS")
-    lines.append("")
-    return lines
-
-
-def cabocha_bunsetu_line(sent: Span, bunsetu_index_list, token):
-    bunsetu_head_index = None
-    bunsetu_dep_index = None
-    bunsetu_func_index = None
-    dep_type = "D"
-    for t in token.doc[token.i:sent.end]:
-        if bunsetu_index_list[t.i] != bunsetu_index_list[token.i]:
-            if bunsetu_func_index is None:
-                bunsetu_func_index = t.i - token.i
-            break
-        tbi = bunsetu_index_list[t.head.i]
-        if bunsetu_index_list[t.i] != tbi:
-            bunsetu_head_index = t.i - token.i
-            bunsetu_dep_index = tbi
-        if bunsetu_func_index is None and bunsetu_position_type(t) in {"FUNC", "SYN_HEAD"}:
-            bunsetu_func_index = t.i - token.i
-    else:
-        if bunsetu_func_index is None:
-            bunsetu_func_index = len(sent) - token.i
-    if bunsetu_head_index is None:
-        bunsetu_head_index = 0
-    if bunsetu_dep_index is None:
-        bunsetu_dep_index = -1
-    return "* {} {}{} {}/{} 0.000000".format(
-        bunsetu_index_list[token.i],
-        bunsetu_dep_index,
-        dep_type,
-        bunsetu_head_index,
-        bunsetu_func_index,
-    )
+        p_analyzes = []
+        abort = Event()
+        for _ in range(parallel_level):
+            p = Process(target=_multi_process_analyze, args=(analyzer, in_queue, out_queue, abort), daemon=True)
+            p.start()
+            p_analyzes.append(p)
 
+        p_load = Process(target=_multi_process_load, args=(in_queue, files, MINI_BATCH_SIZE, parallel_level, abort), daemon=True)
+        p_load.start()
 
-def cabocha_token_line(token):
-    part_of_speech = token.tag_.replace("-", ",")
-    part_of_speech += ",*" * (3 - part_of_speech.count(",")) + "," + inflection(token)
-    reading = reading_form(token)
-    return "{}\t{},{},{},{}\t{}".format(
-        token.orth_,
-        part_of_speech,
-        token.lemma_,
-        reading if reading else token.orth_,
-        "*",
-        "O" if token.ent_iob_ == "O" else "{}-{}".format(token.ent_iob_, token.ent_type_),
-    )
+        _main_process_write(out_queue, output, parallel_level, abort)
 
+    except KeyboardInterrupt:
+        abort.set()
+    finally:
+        for p in [p_load] + p_analyzes:
+            try:
+                p.join(timeout=1)
+            except:
+                if p.is_alive():
+                    p.terminate()
+                    p.join()
+
+
+def _data_loader(files: List[str], batch_size: int) -> Generator[List[str], None, None]:
+    mini_batch = []
+    for path in files:
+        with open(path, "r") as f:
+            for line in f:
+                mini_batch.append(line)
+                if len(mini_batch) == batch_size:
+                    yield mini_batch
+                    mini_batch = []
+    if mini_batch:
+        yield mini_batch
+
+
+def _multi_process_load(in_queue: Queue, files: List[str], batch_size: int, n_analyze_process: int, abort: Event):
+    try:
+        for i, mini_batch in enumerate(_data_loader(files, batch_size)):
+            if abort.is_set():
+                break
+            in_queue.put((i, mini_batch))
+        else:
+            for _ in range(n_analyze_process):
+                in_queue.put("terminate")
+    except KeyboardInterrupt:
+        pass
+    except:
+        traceback.print_exc()
+        abort.set()
 
-def analyze_mecab(sudachipy_tokens):
-    return tuple(mecab_token_line(t) for t in sudachipy_tokens) + ("EOS", "")
 
+def _multi_process_analyze(analyzer: Analyzer, in_queue: Queue, out_queue: Queue, abort: Event):
+    i = None
+    mini_batch = []
+    try:
+        while True:
+            if abort.is_set():
+                break
+            try:
+                msg = in_queue.get(timeout=0.1)
+            except queue.Empty:
+                continue
+            if msg == "terminate":
+                out_queue.put(("terminating", i, None))
+                break
+            i, mini_batch = msg
+            result = analyzer.analyze_batch(mini_batch)
+            out_queue.put((None, i, result))
+    except KeyboardInterrupt:
+        pass
+    except Exception as err:
+        out_queue.put(("Error: {}\n{}".format(err, "".join(mini_batch)), i, None))
+        traceback.print_exc()
+        abort.set()
+
+
+def _main_process_write(out_queue: queue, output: _OutputWrapper, parallel_level: int, abort: Event):
+    cur = 0
+    results = dict()
+    terminating = 0
+    while True:
+        if abort.is_set():
+            return
+        try:
+            msg, mini_batch_index, result = out_queue.get(timeout=0.1)
+        except queue.Empty:
+            continue
+
+        if msg is not None:
+            if msg == "terminating":
+                terminating += 1
+                if terminating == parallel_level:
+                    return
+                continue
+            else:
+                print(f"Analysis failed in mini_batch #{mini_batch_index}. Stopping all the processes.", file=sys.stderr)
+                print(msg, file=sys.stderr)
+                return
 
-def mecab_token_line(token):
-    reading = token.reading_form()
-    return "{}\t{},{},{},{}".format(
-        token.surface(),
-        ",".join(token.part_of_speech()),
-        token.normalized_form(),
-        reading if reading else token.surface(),
-        "*",
-    )
+        # output must be ordered same as input text
+        results[mini_batch_index] = result
+        while results:
+            if cur not in results.keys():
+                break
+            result = results[cur]
+            del results[cur]
+            cur += 1
+            output.write(result)
 
 
 @plac.annotations(
-    model_path=("model directory path", "option", "b", str),
-    split_mode=("split mode", "option", "s", str, ["A", "B", "C", None]),
+    split_mode=("split mode", "option", "s", str, ["A", "B", "C"]),
     hash_comment=("hash comment", "option", "c", str, ["print", "skip", "analyze"]),
     output_path=("output path", "option", "o", Path),
-    use_normalized_form=("overriding Token.lemma_ by normalized_form of SudachiPy", "flag", "n"),
     parallel=("parallel level (default=-1, all_cpus=0)", "option", "p", int),
     files=("input files", "positional"),
 )
 def run_ginzame(
-        model_path=None,
-        split_mode=None,
-        hash_comment="print",
-        output_path=None,
-        use_normalized_form=False,
-        parallel=-1,
-        *files,
+    split_mode=None,
+    hash_comment="print",
+    output_path=None,
+    parallel=-1,
+    *files,
 ):
     run(
-        model_path=model_path,
-        ensure_model="ja_ginza",
+        model_path=None,
+        ensure_model=None,
         split_mode=split_mode,
         hash_comment=hash_comment,
         output_path=output_path,
         output_format="mecab",
-        require_gpu=False,
-        use_normalized_form=use_normalized_form,
-        parallel=parallel,
+        require_gpu=-1,
+        use_normalized_form=True,
+        parallel_level=parallel,
         disable_sentencizer=False,
         files=files,
     )
@@ -440,29 +314,29 @@ def main_ginzame():
 
 @plac.annotations(
     model_path=("model directory path", "option", "b", str),
-    ensure_model=("select model either ja_ginza or ja_ginza_electra", "option", "m", str, ["ja_ginza", "ja-ginza", "ja_ginza_electra", "ja-ginza-electra", None]),
-    split_mode=("split mode", "option", "s", str, ["A", "B", "C", None]),
+    ensure_model=("select model package of ginza or spacy", "option", "m", str),
+    split_mode=("split mode", "option", "s", str, ["A", "B", "C"]),
     hash_comment=("hash comment", "option", "c", str, ["print", "skip", "analyze"]),
     output_path=("output path", "option", "o", Path),
     output_format=("output format", "option", "f", str, ["0", "conllu", "1", "cabocha", "2", "mecab", "3", "json"]),
-    require_gpu=("enable require_gpu", "flag", "g"),
-    use_normalized_form=("overriding Token.lemma_ by normalized_form of SudachiPy", "flag", "n"),
+    require_gpu=("enable require_gpu", "option", "g", int),
+    use_normalized_form=("Use Token.norm_ instead of Token.lemma_", "flag", "n"),
     disable_sentencizer=("disable spaCy's sentence separator", "flag", "d"),
     parallel=("parallel level (default=1, all_cpus=0)", "option", "p", int),
     files=("input files", "positional"),
 )
 def run_ginza(
-        model_path=None,
-        ensure_model=None,
-        split_mode=None,
-        hash_comment="print",
-        output_path=None,
-        output_format="conllu",
-        require_gpu=False,
-        use_normalized_form=False,
-        disable_sentencizer=False,
-        parallel=1,
-        *files,
+    model_path=None,
+    ensure_model=None,
+    split_mode="C",
+    hash_comment="print",
+    output_path=None,
+    output_format="conllu",
+    require_gpu=-1,
+    use_normalized_form=False,
+    disable_sentencizer=False,
+    parallel=1,
+    *files,
 ):
     run(
         model_path=model_path,
@@ -474,7 +348,7 @@ def run_ginza(
         require_gpu=require_gpu,
         use_normalized_form=use_normalized_form,
         disable_sentencizer=disable_sentencizer,
-        parallel=parallel,
+        parallel_level=parallel,
         files=files,
     )
 
diff --git a/ginza/compound_splitter.py b/ginza/compound_splitter.py
index 13c08a2..aa3e8a9 100644
--- a/ginza/compound_splitter.py
+++ b/ginza/compound_splitter.py
@@ -1,11 +1,13 @@
 # encoding: utf8
 from collections import OrderedDict
+import re
 
 import srsly
 
 from spacy import util
 from spacy.language import Language
 from spacy.lang.ja import resolve_pos
+from spacy.tokens import Doc, MorphAnalysis
 
 __all__ = [
     "CompoundSplitter",
@@ -49,11 +51,13 @@ def _replace_list_entries(lst, index, inserting_list):
 
 
 class CompoundSplitter:
-    def __init__(self, nlp, split_mode=None):
-        self.nlp = nlp
+    def __init__(self, vocab, split_mode=None):
+        self.vocab = vocab
         self.split_mode = split_mode
 
-    def __call__(self, doc):
+    def __call__(self, doc: Doc):
+        if "sub_tokens" not in doc.user_data:
+            return doc
         if self._split_mode is None:
             return doc
         elif self._split_mode == "C":
@@ -93,13 +97,24 @@ def __call__(self, doc):
                 else:
                     heads = [(token, last) for _ in range(last)] + [token.head]
                 surfaces = [dtoken.surface for dtoken in sub_tokens]
+                def morph(dtoken):
+                    m = {}
+                    if dtoken.inf:
+                        m["Inflection"] = dtoken.inf
+                    if dtoken.reading:
+                        m["Reading"] = re.sub("[=|]", "_", dtoken.reading)
+                    return "|".join(f"{k}={v}" for k, v in m.items())
                 attrs = {
                     "TAG": [dtoken.tag for dtoken in sub_tokens],
                     "DEP": deps,
                     "POS": tag_to_pos(
                         sub_tokens,
                         doc[token.i + 1].tag_ if token.i < len(doc) - 1 else None
-                    )
+                    ),
+                    "LEMMA": [dtoken.lemma for dtoken in sub_tokens],
+                    "NORM": [dtoken.norm for dtoken in sub_tokens],
+                    "ENT_TYPE": [token_ent_type for dtoken in sub_tokens],
+                    "MORPH": [morph(dtoken) for dtoken in sub_tokens],
                 }
                 try:
                     with doc.retokenize() as retokenizer:
@@ -119,29 +134,6 @@ def __call__(self, doc):
                             if t.head.i == token_i:
                                 t.head = doc[token_i + last]
 
-                for t, dtoken in zip(
-                    doc[token_i:token_i + len(sub_tokens)],
-                    sub_tokens
-                ):
-                    t.lemma_ = dtoken.lemma
-
-                if token_ent_type:  # work-around: _retokenize() sets value for Token.ent_iob but not for ent_type
-                    for t in doc[token_i + 1:token_i + len(sub_tokens)]:
-                        t.ent_type = token_ent_type
-
-                if "inflections" in doc.user_data:
-                    doc.user_data["inflections"] = _replace_list_entries(
-                        doc.user_data["inflections"],
-                        token_i,
-                        tuple(dtoken.inf for dtoken in sub_tokens),
-                    )
-                if "reading_forms" in doc.user_data:
-                    doc.user_data["reading_forms"] = _replace_list_entries(
-                        doc.user_data["reading_forms"],
-                        token_i,
-                        tuple(dtoken.reading for dtoken in sub_tokens),
-                    )
-
         del doc.user_data["sub_tokens"]
         return doc
 
diff --git a/ginza/disable_sentencizer.py b/ginza/disable_sentencizer.py
index 4bcf747..2dd6bd1 100644
--- a/ginza/disable_sentencizer.py
+++ b/ginza/disable_sentencizer.py
@@ -17,7 +17,7 @@ def __init__(self, nlp):
         self.nlp = nlp
 
     def __call__(self, doc):
-        for t in doc:
+        for t in doc[1:]:
             t.is_sent_start = False
         return doc
 
diff --git a/meta.json.ginza b/meta.json.ginza
deleted file mode 100644
index 063fabb..0000000
--- a/meta.json.ginza
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-  "author":"Megagon Labs, Tokyo.",
-  "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ v2.6 + GSK2014-A(2019). Assigns word2vec token vectors, POS tags, dependency parse and named entities.",
-  "email":"ginza@megagon.ai",
-  "lang":"ja",
-  "license":"MIT",
-  "name":"ginza",
-  "parent_package":"spacy",
-  "requirements": [
-  ],
-  "sources":[
-    "SudachiPy",
-    "SudachiDict_core",
-    "jawiki-cirrussearch-content",
-    "UD-Japanese BCCWJ v2.6",
-    "GSK2014-A(2019)"
-  ],
-  "url":"https://github.com/megagonlabs/ginza"
-}
diff --git a/requirements.txt b/requirements.txt
index dc02956..52ee66b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-spacy>=3.0.6,<3.2.0
+spacy>=3.2.0,<3.3.0
 plac>=1.3.3
-SudachiPy>=0.5.2,<0.6.0
-SudachiDict-core>=20210608
+SudachiPy>=0.6.2,<0.7.0
+SudachiDict-core>=20210802
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..31ad82b
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,2 @@
+[aliases]
+test = pytest
diff --git a/setup.py b/setup.py
index fbeaf4f..3583fda 100644
--- a/setup.py
+++ b/setup.py
@@ -17,14 +17,16 @@
         ],
     },
     install_requires=[
-        "spacy>=3.0.6,<3.2.0",
+        "spacy>=3.2.0,<3.3.0",
         "plac>=1.3.3",
-        "SudachiPy>=0.5.2,<0.6.0",
-        "SudachiDict-core>=20210608",
+        "SudachiPy>=0.6.2,<0.7.0",
+        "SudachiDict-core>=20210802",
     ],
+    setup_requires=["pytest-runner"],
+    tests_require=["pytest", "pytest-cov", "pytest-mock"],
     license="MIT",
     name="ginza",
     packages=find_packages(include=["ginza"]),
     url="https://github.com/megagonlabs/ginza",
-    version='5.0.3',
+    version='5.1.0',
 )


Test patch:
diff --git a/ginza/tests/conftest.py b/ginza/tests/conftest.py
new file mode 100644
index 0000000..0ea898e
--- /dev/null
+++ b/ginza/tests/conftest.py
@@ -0,0 +1,15 @@
+import tempfile
+import subprocess as sp
+import sys
+from pathlib import Path
+from functools import partial
+
+import pytest
+
+run_cmd = partial(sp.run, encoding="utf-8", stdout=sp.PIPE)
+
+
+@pytest.fixture(scope="session")
+def tmpdir() -> Path:
+    with tempfile.TemporaryDirectory() as dir_name:
+        yield Path(dir_name)
diff --git a/ginza/tests/test_analyzer.py b/ginza/tests/test_analyzer.py
new file mode 100644
index 0000000..df857bd
--- /dev/null
+++ b/ginza/tests/test_analyzer.py
@@ -0,0 +1,146 @@
+
+import json
+import pytest
+
+from ginza.analyzer import Analyzer
+
+
+TOKEN_TESTS = [
+    ["今日はかつ丼を食べた。明日は蕎麦を食べたい。", ["今日","は","かつ丼","を","食べ","た","。","明日","は","蕎麦","を","食べ","たい","。"]]
+]
+
+MECAB_TESTS = [
+    ["今日はかつ丼を食べた。明日は蕎麦を食べたい。", ["今日","は","かつ","丼","を","食べ","た","。","明日","は","蕎麦","を","食べ","たい","。"]]
+]
+
+BATCH_TESTS = [
+    [
+        [
+            "銀座でランチをご一緒しましょう。",
+            "東京タワーの近くに住んでいます。",
+            "東京都選挙管理委員会の担当者は、次のように説明した。",
+        ], [
+            ["銀座","で","ランチ","を","ご","一緒","し","ましょう","。"],
+            ["東京","タワー","の","近く","に","住ん","で","い","ます","。"],
+            ["東京都","選挙管理委員会","の","担当者","は","、","次","の","よう","に","説明","し","た","。"],
+        ]
+    ]
+]
+
+@pytest.fixture
+def analyzer() -> Analyzer:
+    default_params = dict(
+        model_name_or_path=None,
+        split_mode=None,
+        hash_comment="print",
+        output_format="conllu",
+        require_gpu=-1,
+        disable_sentencizer=False,
+        use_normalized_form=False,
+    )
+    yield Analyzer(**default_params)
+
+
+def _tokens_conllu(result: str):
+    ret = []
+    for line in result.split("\n"):
+        if line.startswith("#") or line.strip() == "":
+            continue
+        ret.append(line.split("\t")[1])
+    return ret
+
+
+def _tokens_cabocha(result: str):
+    ret = []
+    for line in result.split("\n"):
+        if line.startswith("*") or line.strip() in ("","EOS"):
+            continue
+        ret.append(line.split("\t")[0])
+    return ret
+
+
+def _tokens_mecab(result: str):
+    ret = []
+    for line in result.split("\n"):
+        if line.startswith("#") or line.strip() in ("","EOS"):
+            continue
+        ret.append(line.split("\t")[0])
+    return ret
+
+
+def _tokens_json(result: str):
+    data = json.loads(f"[{result}]")
+    ret = []
+    for d in data:
+        for p in d["paragraphs"]:
+            for s in p["sentences"]:
+                for t in s["tokens"]:
+                    ret.append(t["orth"])
+    return ret
+
+class TestAnalyzer:
+    def test_model_name_or_path_ja_ginza(self, mocker, analyzer):
+        spacy_load_mock = mocker.patch("spacy.load")
+        analyzer.model_name_or_path = "ja_ginza"
+        analyzer.set_nlp()
+        spacy_load_mock.assert_called_once_with("ja_ginza")
+
+    def test_model_name_or_path_ja_ginza_electra(self, mocker, analyzer):
+        spacy_load_mock = mocker.patch("spacy.load")
+        analyzer.model_name_or_path = "ja_ginza_electra"
+        analyzer.set_nlp()
+        spacy_load_mock.assert_called_once_with("ja_ginza_electra")
+
+    def test_require_gpu(self, mocker, analyzer):
+        require_gpu_mock = mocker.patch("thinc.api.require_gpu")
+        analyzer.require_gpu = 0
+        analyzer.set_nlp()
+        require_gpu_mock.assert_called_once()
+
+    @pytest.mark.parametrize("input_text, tokens", TOKEN_TESTS)
+    @pytest.mark.parametrize(
+        "output_format, raises_analysis_before_set, tokens_func",
+        [
+            ("conllu", TypeError, _tokens_conllu),
+            ("cabocha", TypeError, _tokens_cabocha),
+            ("json", TypeError, _tokens_json),
+        ],
+    )
+    def test_analyze_line(self, output_format, raises_analysis_before_set, input_text, tokens, tokens_func, analyzer):
+        analyzer.output_format = output_format
+        with pytest.raises(raises_analysis_before_set):
+            analyzer.analyze_line(input_text)
+
+        analyzer.set_nlp()
+        ret = analyzer.analyze_line(input_text)
+        assert tokens_func(ret) == tokens
+
+    @pytest.mark.parametrize("input_text, tokens", MECAB_TESTS)
+    @pytest.mark.parametrize(
+        "output_format, raises_analysis_before_set, tokens_func",
+        [
+            ("mecab", AttributeError, _tokens_mecab),
+        ],
+    )
+    def test_analyze_line_mecab(self, output_format, raises_analysis_before_set, input_text, tokens, tokens_func, analyzer):
+        analyzer.output_format = output_format
+        with pytest.raises(raises_analysis_before_set):
+            analyzer.analyze_line(input_text)
+
+        analyzer.set_nlp()
+        ret = analyzer.analyze_line(input_text)
+        assert tokens_func(ret) == tokens
+
+    @pytest.mark.parametrize("input_batch, tokens_batch", BATCH_TESTS)
+    @pytest.mark.parametrize(
+        "output_format, tokens_func",
+        [
+            ("conllu", _tokens_conllu),
+            ("cabocha", _tokens_cabocha),
+            ("json", _tokens_json),
+        ],
+    )
+    def test_analyze_batch(self, output_format, input_batch, tokens_batch, tokens_func, analyzer):
+        analyzer.output_format = output_format
+        ret = analyzer.analyze_batch(input_batch)
+        assert tokens_func(ret) == sum(tokens_batch, [])
diff --git a/ginza/tests/test_command_line.py b/ginza/tests/test_command_line.py
new file mode 100644
index 0000000..9dd2b19
--- /dev/null
+++ b/ginza/tests/test_command_line.py
@@ -0,0 +1,327 @@
+import json
+import os
+import subprocess as sp
+from functools import partial
+from pathlib import Path
+from typing import Iterable, List
+
+import pytest
+
+import torch
+import ginza.command_line as cli
+
+TEST_TEXT = "#コメント\n今日はかつ丼を食べた。\n明日は東京で蕎麦を食べる。明後日は酒が飲みたい。"
+
+run_cmd = partial(sp.run, encoding="utf-8", stdout=sp.PIPE)
+
+
+@pytest.fixture(scope="module")
+def input_file(tmpdir: Path) -> Path:
+    file_path = (tmpdir / "test_input.txt").resolve()
+    with open(file_path, "w") as fp:
+        print(TEST_TEXT, file=fp)
+    yield file_path
+    file_path.unlink()
+
+
+@pytest.fixture(scope="module")
+def input_files(tmpdir: Path) -> Iterable[Path]:
+    paths = []
+    for i, text in enumerate(TEST_TEXT.split("\n")):
+        file_path = (tmpdir / f"test_input_{i}.txt").resolve()
+        with open(file_path, "w") as fp:
+            print(text, file=fp)
+        paths.append(file_path)
+    yield paths
+    for file_path in paths:
+        file_path.unlink()
+
+
+@pytest.fixture(scope="module")
+def long_input_file(tmpdir: Path) -> Iterable[Path]:
+    file_path = (tmpdir / "test_long_input.txt").resolve()
+    with open(file_path, "w") as fp:
+        for _ in range(10):
+            print(TEST_TEXT, file=fp)
+    yield file_path
+    file_path.unlink()
+
+
+@pytest.fixture
+def output_file(tmpdir: Path) -> Path:
+    file_path = (tmpdir / "test_output.txt").resolve()
+    file_path.touch()
+    yield file_path
+    file_path.unlink()
+
+
+def _conllu_parsable(result: str):
+    for line in result.split("\n"):
+        if line.startswith("#") or line.strip() == "":
+            continue
+        if not len(line.strip().split("\t")) == 10:
+            raise Exception
+
+
+def _cabocha_parsable(result: str):
+    for line in result.split("\n"):
+        if line.strip() in ("", "EOS") or line.startswith("*") or line.startswith("#"):
+            continue
+        if not len(line.split("\t")) == 3:
+            raise Exception
+        if not len(line.split("\t")[1].split(",")) in [8, 9]:
+            raise Exception
+
+
+def _mecab_parsable(result: str):
+    for line in result.split("\n"):
+        if line.strip() in ("", "EOS") or line.startswith("#"):
+            continue
+        if not len(line.split("\t")) == 2:
+            raise Exception
+        if not len(line.split("\t")[1].split(",")) == 9:
+            raise Exception
+
+
+def _json_parsable(result: str):
+    data = json.loads(result)
+    for d in data:
+        if not type(d) == dict:
+            raise Exception
+        if not "paragraphs" in d.keys():
+            raise Exception
+
+
+class TestCLIGinza:
+    def test_help(self):
+        for opt in ["-h", "--help"]:
+            p = run_cmd(["ginza", opt])
+            assert p.returncode == 0
+
+    def test_input(self, input_file):
+        # input file
+        p = run_cmd(["ginza", input_file])
+
+        # input from stdin
+        p_stdin = sp.Popen(["ginza"], stdin=sp.PIPE, stdout=sp.PIPE)
+        o, e = p_stdin.communicate(input=TEST_TEXT.encode())
+        assert e is None
+        assert o.decode("utf-8") == p.stdout
+
+    def test_multiple_input(self, input_files, input_file):
+        p_multi = run_cmd(["ginza", *input_files])
+        assert p_multi.returncode == 0
+
+        p_single = run_cmd(["ginza", input_file])
+        assert p_multi.stdout == p_single.stdout
+
+    # TODO: add user defined model to fixture and test it here
+    @pytest.mark.parametrize(
+        "model_path, exit_ok",
+        [
+            ("ja_ginza", True),
+            ("not-exist-model", False),
+        ],
+    )
+    def test_model_path(self, model_path, exit_ok, input_file):
+        p = run_cmd(["ginza", "-b", model_path, input_file])
+        assert (p.returncode == 0) is exit_ok
+
+    @pytest.mark.parametrize(
+        "ensure_model, exit_ok",
+        [
+            ("ja_ginza", True),
+            ("ja-ginza", True),
+            ("ja-ginza-electra", True),
+            ("ja_ginza_electra", True),
+            ("ja-ginza_electra", True),
+            ("not-exist-model", False),
+        ],
+    )
+    def test_ensure_model(self, ensure_model, exit_ok, input_file):
+        p = run_cmd(["ginza", "-m", ensure_model, input_file])
+        assert (p.returncode == 0) is exit_ok
+
+    def test_double_model_spcification(self, input_file):
+        p = run_cmd(["ginza", "-b", "ja_ginza", "-m", "ja_ginza", input_file])
+        assert p.returncode != 0
+
+    @pytest.mark.parametrize(
+        "split_mode, input_text, expected",
+        [
+            ("A", "機能性食品", ["機能", "性", "食品"]),
+            ("B", "機能性食品", ["機能性", "食品"]),
+            ("C", "機能性食品", ["機能性食品"]),
+        ],
+    )
+    def test_split_mode(self, split_mode, input_text, expected):
+        p = run_cmd(["ginza", "-s", split_mode], input=input_text)
+        assert p.returncode == 0
+
+        def _sub_words(lines: Iterable) -> List[str]:
+            return [l.split("\t")[1] for l in lines if len(l.split("\t")) > 1]
+
+        assert _sub_words(p.stdout.split("\n")) == expected
+
+    @pytest.mark.parametrize(
+        "hash_comment, n_sentence, n_analyzed_sentence, exit_ok",
+        [
+            ("print", 4, 3, True),
+            ("skip", 3, 3, True),
+            ("analyze", 4, 4, True),
+        ],
+    )
+    def test_hash_comment(self, hash_comment, n_sentence, n_analyzed_sentence, exit_ok, input_file):
+        def _n_sentence(lines: Iterable) -> int:
+            return len(list(filter(lambda x: x.startswith("#"), lines)))
+
+        def _n_analyzed_sentence(lines: Iterable) -> int:
+            return len(list(filter(lambda x: x.startswith("# text = "), lines)))
+
+        p = run_cmd(["ginza", "-c", hash_comment, input_file])
+        assert (p.returncode == 0) is exit_ok
+        assert _n_sentence(p.stdout.split("\n")) == n_sentence
+        assert _n_analyzed_sentence(p.stdout.split("\n")) == n_analyzed_sentence
+
+    def test_output_path(self, input_file, output_file):
+        p_s = run_cmd(["ginza", input_file])
+        p_o = run_cmd(["ginza", "-o", output_file, input_file])
+        assert p_o.returncode == 0
+
+        def _file_output():
+            with open(output_file, "r") as fp:
+                return [l.strip() for l in fp if l.strip()]
+
+        def _pipe_output():
+            return [l.strip() for l in p_s.stdout.split("\n") if l.strip()]
+
+        assert _file_output() == _pipe_output()
+
+    @pytest.mark.parametrize(
+        "output_format, result_parsable",
+        [
+            ("conllu", _conllu_parsable),
+            ("cabocha", _cabocha_parsable),
+            ("mecab", _mecab_parsable),
+            ("json", _json_parsable),
+        ],
+    )
+    def test_output_format(self, output_format, result_parsable, input_file):
+        p = run_cmd(["ginza", "-c", "analyze", "-f", output_format, input_file])
+        assert p.returncode == 0
+        result_parsable(p.stdout.strip())
+
+    @pytest.mark.parametrize(
+        "hash_comment", ["print", "skip"]
+    )
+    def test_warn_if_json_hash_comment_not_analyze(self, hash_comment, input_file):
+        p = run_cmd(["ginza", "-c", hash_comment, "-f", "json", input_file], stderr=sp.PIPE)
+        assert p.returncode == 0
+        msg = (
+            f'hash_comment="{hash_comment}" not permitted for JSON output. Forced to use hash_comment="analyze"'
+        )
+        assert msg in p.stderr
+
+    def test_require_gpu(self, input_file):
+        p = run_cmd(["ginza", "-g", "0", input_file])
+        gpu_available = torch.cuda.is_available()
+        assert (p.returncode == 0) is gpu_available
+
+    def test_do_not_use_normalized_form(self, input_file):
+        p = run_cmd(["ginza", input_file])
+        lemmas = [l.split("\t")[2] for l in p.stdout.split("\n") if len(l.split("\t")) > 1]
+        # 'かつ丼' is dictionary_form of 'かつ丼'
+        assert p.returncode == 0
+        assert "かつ丼" in lemmas
+
+    def test_use_normalized_form(self, input_file):
+        p = run_cmd(["ginza", "-n", input_file])
+        lemmas = [l.split("\t")[2] for l in p.stdout.split("\n") if len(l.split("\t")) > 1]
+        # 'カツ丼' is normlized_form of 'かつ丼'
+        assert p.returncode == 0
+        assert "カツ丼" in lemmas
+
+    def test_disable_sentencizer(self, input_file):
+        p = run_cmd(["ginza", "-d", input_file])
+
+        def _n_analyzed_sentence(lines: Iterable) -> int:
+            return len(list(filter(lambda x: x.startswith("# text = "), lines)))
+
+        assert p.returncode == 0
+        assert _n_analyzed_sentence(p.stdout.split("\n")) == 2
+
+    def test_parallel(self, input_file):
+        p = run_cmd(["ginza", "-p", "2", input_file])
+        assert p.returncode == 0
+
+
+class TestCLIGinzame:
+    def test_ginzame(self, input_file):
+        p_ginzame = run_cmd(["ginzame", input_file])
+        p_ginza = run_cmd(["ginza", "-n", "-m", "ja_ginza", "-f", "2", "-s", "A", input_file])
+
+        assert p_ginzame.returncode == 0
+        assert p_ginzame.stdout == p_ginza.stdout
+
+
+class TestRun:
+    def test_run_as_single_when_input_is_a_tty(self, mocker, output_file, long_input_file):
+        i = 0
+
+        def f_mock_input():
+            nonlocal i
+            if i >= 1:
+                raise KeyboardInterrupt
+            else:
+                i += 1
+                return "今日はいい天気だ"
+
+        mocker.patch.object(cli, "MINI_BATCH_SIZE", 5)
+        mocker.patch("ginza.command_line.sys.stdin.isatty", return_value=True)
+        input_mock = mocker.patch.object(cli, "input", side_effect=f_mock_input)
+        analyze_parallel_mock = mocker.patch.object(cli, "_analyze_parallel")
+        cli.run(parallel_level=2, output_path=output_file, files=None)
+        assert input_mock.call_count == 2
+        analyze_parallel_mock.assert_not_called()
+
+    @pytest.mark.parametrize(
+        "output_format",
+        ["conllu", "cabocha", "mecab", "json"],
+    )
+    def test_parallel_output_same_as_single(self, output_format, mocker, tmpdir, long_input_file):
+        mocker.patch.object(cli, "MINI_BATCH_SIZE", 5)
+
+        out_single = tmpdir / "single_output.txt"
+        if out_single.exists():
+            out_single.unlink()
+        cli.run(
+            parallel_level=1,
+            output_path=out_single,
+            output_format=output_format,
+            files=[long_input_file],
+            ensure_model="ja_ginza",
+        )
+
+        out_parallel = tmpdir / "parallel_output.txt"
+        if out_parallel.exists():
+            out_parallel.unlink()
+        try:
+            cli.run(
+                parallel_level=2,
+                output_path=out_parallel,
+                output_format=output_format,
+                files=[long_input_file],
+                ensure_model="ja_ginza",
+            )
+        except:
+            pytest.fail("parallel run failed")
+
+        def f_len(path):
+            with open(path, "r") as f:
+                return sum([1 for _ in f])
+
+        assert f_len(out_single) == f_len(out_parallel)
+        with open(out_single, "r") as f_s:
+            with open(out_parallel, "r") as f_p:
+                for s, p in zip(f_s, f_p):
+                    assert s == p
diff --git a/ginza/tests/test_models.py b/ginza/tests/test_models.py
new file mode 100644
index 0000000..27b0951
--- /dev/null
+++ b/ginza/tests/test_models.py
@@ -0,0 +1,230 @@
+import spacy
+import pytest
+from copy import deepcopy
+
+from ginza import set_split_mode
+
+
+MODELS = ["ja_ginza", "ja_ginza_electra"]
+
+TOKENIZER_TESTS = [
+    ("銀座でランチをご一緒しましょう。", ["銀座", "で", "ランチ", "を", "ご", "一緒", "し", "ましょう", "。"]),
+    ("すもももももももものうち", ["すもも", "も", "もも", "も", "もも", "の", "うち"]),
+]
+
+COMPOUND_SPLITER_TESTS = [
+    ("選挙管理委員会", 4, 3, 1),
+    ("客室乗務員", 3, 2, 1),
+    ("労働者協同組合", 4, 3, 1),
+    ("機能性食品", 3, 2, 1),
+]
+
+TAG_TESTS = [
+    ("銀座でランチをご一緒しましょう。", ["名詞-固有名詞-地名-一般", "助詞-格助詞", "名詞-普通名詞-一般", "助詞-格助詞", "接頭辞", "名詞-普通名詞-サ変可能", "動詞-非自立可能", "助動詞", "補助記号-句点"]),
+    ("すもももももももものうち", ["名詞-普通名詞-一般", "助詞-係助詞", "名詞-普通名詞-一般", "助詞-係助詞", "名詞-普通名詞-一般", "助詞-格助詞", "名詞-普通名詞-副詞可能"]),
+]
+
+POS_TESTS_JA_GINZA = [
+    ("銀座でランチをご一緒しましょう。", ["PROPN", "ADP", "NOUN", "ADP", "NOUN", "NOUN", "AUX", "AUX", "PUNCT"]),
+    ("すもももももももものうち", ["NOUN", "ADP", "NOUN", "ADP", "NOUN", "ADP", "NOUN"]),
+]
+
+POS_TESTS_JA_GINZA_ELECTRA = [
+    ("銀座でランチをご一緒しましょう。", ["PROPN", "ADP", "NOUN", "ADP", "NOUN", "VERB", "AUX", "AUX", "PUNCT"]),
+    ("すもももももももものうち", ["NOUN", "ADP", "NOUN", "ADP", "NOUN", "ADP", "NOUN"]),
+]
+
+LEMMATIZE_TESTS = [
+    ("新しく", "新しい"),
+    ("いただきました", "いただく"),
+    ("なった", "なる"),
+]
+
+NORMALIZE_TESTS = [
+    ("かつ丼", "カツ丼"),
+    ("附属", "付属"),
+    ("SUMMER", "サマー"),
+    ("シュミレーション", "シミュレーション"),
+]
+
+EMPTYISH_TESTS = [
+    ("", 0),
+    ("         ", 1),
+    ("\n\n\t\t\n\n", 1),
+    ("\r\n\r\n", 1),
+    ("\n&nbsp;\n\n", 5),
+]
+
+NAUGHTY_STRINGS = [
+    # ASCII punctuation
+    r",./;'[]\-=",
+    r'<>?:"{}|_+',
+    r'!@#$%^&*()`~"',
+    # Unicode additional control characters, byte order marks
+    r"­؀؁؂؃؄؅؜۝܏᠎​‌‍‎‏‪",
+    r"￾",
+    # Unicode Symbols
+    r"Ω≈ç√∫˜µ≤≥÷",
+    r"åß∂ƒ©˙∆˚¬…æ",
+    "œ∑´®†¥¨ˆøπ“‘",
+    r"¡™£¢∞§¶•ªº–≠",
+    r"¸˛Ç◊ı˜Â¯˘¿",
+    r"ÅÍÎÏ˝ÓÔÒÚÆ☃",
+    r"Œ„´‰ˇÁ¨ˆØ∏”’",
+    r"`⁄€‹›ﬁﬂ‡°·‚—±",
+    r"⅛⅜⅝⅞",
+    r"ЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя",
+    r"٠١٢٣٤٥٦٧٨٩",
+    # Unicode Subscript/Superscript/Accents
+    r"⁰⁴⁵",
+    r"₀₁₂",
+    r"⁰⁴⁵₀₁₂",
+    r"ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็",
+    r" ̄  ̄",
+    # Two-Byte Characters
+    r"田中さんにあげて下さい",
+    r"パーティーへ行かないか",
+    r"和製漢語",
+    r"部落格",
+    r"사회과학원 어학연구소",
+    r"찦차를 타고 온 펲시맨과 쑛다리 똠방각하",
+    r"社會科學院語學研究所",
+    r"울란바토르",
+    r"𠜎𠜱𠝹𠱓𠱸𠲖𠳏",
+    # Japanese Emoticons
+    r"ヽ༼ຈل͜ຈ༽ﾉ ヽ༼ຈل͜ຈ༽ﾉ",
+    r"(｡◕ ∀ ◕｡)",
+    r"｀ｨ(´∀｀∩",
+    r"__ﾛ(,_,*)",
+    r"・(￣∀￣)・:*:",
+    r"ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ",
+    r",。・:*:・゜’( ☻ ω ☻ )。・:*:・゜’",
+    r"(╯°□°）╯︵ ┻━┻)" "(ﾉಥ益ಥ）ﾉ﻿ ┻━┻",
+    r"┬─┬ノ( º _ ºノ)",
+    r"( ͡° ͜ʖ ͡°)",
+    # Emoji
+    r"😍",
+    r"👩🏽",
+    r"👾 🙇 💁 🙅 🙆 🙋 🙎 🙍",
+    r"🐵 🙈 🙉 🙊",
+    r"❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙",
+    r"✋🏿 💪🏿 👐🏿 🙌🏿 👏🏿 🙏🏿",
+    r"🚾 🆒 🆓 🆕 🆖 🆗 🆙 🏧",
+    r"0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟",
+    # Regional Indicator Symbols
+    r"🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸",
+    r"🇺🇸🇷🇺🇸🇦🇫🇦🇲",
+    r"🇺🇸🇷🇺🇸🇦",
+    # Unicode Numbers
+    r"１２３",
+    r"١٢٣",
+    # Right-To-Left Strings
+    r"ثم نفس سقطت وبالتحديد،, جزيرتي باستخدام أن دنو. إذ هنا؟ الستار وتنصيب كان. أهّل ايطاليا، بريطانيا-فرنسا قد أخذ. سليمان، إتفاقية بين ما, يذكر الحدود أي بعد, معاملة بولندا، الإطلاق عل إيو.",
+    r"إيو.",
+    r"בְּרֵאשִׁית, בָּרָא אֱלֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָאָרֶץ",
+    r"הָיְתָהtestالصفحات التّحول",
+    r"﷽",
+    r"ﷺ",
+    r"مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ،",
+    # Trick Unicode
+    r"‪‪test‪",
+    r"‫test",
+    r" test ",
+    r"test⁠test",
+    r"⁦test⁧",
+    # Zalgo Text
+    r"Ṱ̺̺̕o͞ ̷i̲̬͇̪͙n̝̗͕v̟̜̘̦͟o̶̙̰̠kè͚̮̺̪̹̱̤ ̖t̝͕̳̣̻̪͞h̼͓̲̦̳̘̲e͇̣̰̦̬͎ ̢̼̻̱̘h͚͎͙̜̣̲ͅi̦̲̣̰̤v̻͍e̺̭̳̪̰-m̢iͅn̖̺̞̲̯̰d̵̼̟͙̩̼̘̳ ̞̥̱̳̭r̛̗̘e͙p͠r̼̞̻̭̗e̺̠̣͟s̘͇̳͍̝͉e͉̥̯̞̲͚̬͜ǹ̬͎͎̟̖͇̤t͍̬̤͓̼̭͘ͅi̪̱n͠g̴͉ ͏͉ͅc̬̟h͡a̫̻̯͘o̫̟̖͍̙̝͉s̗̦̲.̨̹͈̣",
+    r"̡͓̞ͅI̗̘̦͝n͇͇͙v̮̫ok̲̫̙͈i̖͙̭̹̠̞n̡̻̮̣̺g̲͈͙̭͙̬͎ ̰t͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨o͚̪͡f̘̣̬ ̖̘͖̟͙̮c҉͔̫͖͓͇͖ͅh̵̤̣͚͔á̗̼͕ͅo̼̣̥s̱͈̺̖̦̻͢.̛̖̞̠̫̰",
+    r"̗̺͖̹̯͓Ṯ̤͍̥͇͈h̲́e͏͓̼̗̙̼̣͔ ͇̜̱̠͓͍ͅN͕͠e̗̱z̘̝̜̺͙p̤̺̹͍̯͚e̠̻̠͜r̨̤͍̺̖͔̖̖d̠̟̭̬̝͟i̦͖̩͓͔̤a̠̗̬͉̙n͚͜ ̻̞̰͚ͅh̵͉i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟",
+    r"̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕",
+    r"Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮",
+    # Unicode Upsidedown
+    r"˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥",
+    r"00˙Ɩ$-",
+    # Unicode font
+    r"Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ",
+    r"𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠",
+    r"𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌",
+    r"𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈",
+    r"𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰",
+    r"𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘",
+    r"𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐",
+    r"⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢",
+    # File paths
+    r"../../../../../../../../../../../etc/passwd%00",
+    r"../../../../../../../../../../../etc/hosts",
+    # iOS Vulnerabilities
+    r"Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗",
+    r"🏳0🌈️",
+]
+
+
+@pytest.fixture(scope="module")
+def nlp(request):
+    return spacy.load(request.param)
+
+
+@pytest.mark.parametrize("nlp", MODELS, indirect=True)
+@pytest.mark.parametrize("text, expected_tokens", TOKENIZER_TESTS)
+def test_tokenize(nlp, text, expected_tokens):
+    tokens = [token.text for token in nlp(text)]
+    assert tokens == expected_tokens
+
+
+@pytest.mark.parametrize("nlp", MODELS, indirect=True)
+@pytest.mark.parametrize("text, len_a, len_b, len_c", COMPOUND_SPLITER_TESTS)
+def test_compound_spliter(nlp, text, len_a, len_b, len_c):
+    assert len(nlp(text)) == len_c
+    for split_mode, l in zip(["A", "B", "C"], [len_a, len_b, len_c]):
+        nlp = deepcopy(nlp)
+        set_split_mode(nlp, split_mode)
+        assert len(nlp(text)) == l
+
+
+@pytest.mark.parametrize("nlp", MODELS, indirect=True)
+@pytest.mark.parametrize("text, expected_tags", TAG_TESTS)
+def test_tag(nlp, text, expected_tags):
+    tags = [token.tag_ for token in nlp(text)]
+    assert tags == expected_tags
+
+
+@pytest.mark.parametrize("nlp", ["ja_ginza"], indirect=True)
+@pytest.mark.parametrize("text, expected_poss", POS_TESTS_JA_GINZA)
+def test_pos_ja_ginza(nlp, text, expected_poss):
+    poss = [token.pos_ for token in nlp(text)]
+    assert poss == expected_poss
+
+
+@pytest.mark.parametrize("nlp", ["ja_ginza_electra"], indirect=True)
+@pytest.mark.parametrize("text, expected_poss", POS_TESTS_JA_GINZA_ELECTRA)
+def test_pos_ja_ginza_electra(nlp, text, expected_poss):
+    poss = [token.pos_ for token in nlp(text)]
+    assert poss == expected_poss
+
+
+@pytest.mark.parametrize("nlp", MODELS, indirect=True)
+@pytest.mark.parametrize("text, lemma", LEMMATIZE_TESTS)
+def test_lemmatize(nlp, text, lemma):
+    doc = nlp(text)
+    assert lemma == doc[0].lemma_
+
+
+@pytest.mark.parametrize("nlp", MODELS, indirect=True)
+@pytest.mark.parametrize("text, norm", NORMALIZE_TESTS)
+def test_normalize(nlp, text, norm):
+    doc = nlp(text)
+    assert norm == doc[0].norm_
+
+
+@pytest.mark.parametrize("nlp", MODELS, indirect=True)
+@pytest.mark.parametrize("text, expected_len", EMPTYISH_TESTS)
+def test_emptyish_texts(nlp, text, expected_len):
+    doc = nlp(text)
+    assert len(doc) == expected_len
+
+
+@pytest.mark.parametrize("nlp", MODELS, indirect=True)
+@pytest.mark.parametrize("text", NAUGHTY_STRINGS)
+def test_naughty_strings(nlp, text):
+    doc = nlp(text)
+    assert doc.text_with_ws == text