Repository: WorksApplications/sudachi.rs
Pull request #33 (2021-09-22 09:19:09)

Remove lifetimes from Dictionary
This is needed to use Pyo3.

Also, the new version of dictionary will use memory mapped files for loading dictionaries.

eiennohito: What remains: replace old implementation with the new one (for the time being).
eiennohito: @mh-northlander Please squash commits into a single one when merging (or do only review, I will merge it afterwards itself).

Patch:
diff --git a/Cargo.lock b/Cargo.lock
index d4d7d4a..1d0ba3e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -203,6 +203,15 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
 
+[[package]]
+name = "memmap2"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4647a11b578fead29cdbb34d4adef8dd3dc35b876c9c6d5240d83f205abfe96e"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.1.3"
@@ -441,6 +450,7 @@ dependencies = [
  "fancy-regex",
  "lazy_static",
  "libloading",
+ "memmap2",
  "multiset",
  "nom",
  "serde",
diff --git a/sudachi-cli/src/main.rs b/sudachi-cli/src/main.rs
index 29e53be..8582600 100644
--- a/sudachi-cli/src/main.rs
+++ b/sudachi-cli/src/main.rs
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-use std::borrow::Cow;
 use std::fs::File;
 use std::io::{self, BufRead, BufReader, BufWriter, Write};
 use std::path::PathBuf;
@@ -23,7 +22,9 @@ use std::process;
 use structopt::StructOpt;
 
 use sudachi::config::Config;
+use sudachi::dic::dictionary::JapaneseDictionary;
 use sudachi::prelude::*;
+use sudachi::stateless_tokeniser::StatelessTokenizer;
 
 #[cfg(feature = "bake_dictionary")]
 const BAKED_DICTIONARY_BYTES: &[u8] = include_bytes!(env!("SUDACHI_DICT_PATH"));
@@ -108,24 +109,15 @@ fn main() {
         args.resource_dir.clone(),
         args.dictionary_path.clone(),
     )
-        .expect("Failed to load config file");
-
-    // load and parse dictionary binary to create a tokenizer
-    let dictionary_bytes =
-        get_dictionary_bytes(config.system_dict.as_ref()).expect("No system dictionary found");
-    let mut user_dictionary_bytes = Vec::with_capacity(config.user_dicts.len());
-    for pb in &config.user_dicts {
-        let storage_buf =
-            dictionary_bytes_from_path(pb).expect("Failed to get user dictionary bytes from file");
-        user_dictionary_bytes.push(storage_buf.into_boxed_slice());
-    }
-    let tokenizer =
-        Tokenizer::from_dictionary_bytes(&dictionary_bytes, &user_dictionary_bytes, &config)
-            .expect("Failed to create Tokenizer from dictionary bytes");
+    .expect("Failed to load config file");
+
+    let dict = JapaneseDictionary::from_cfg(&config)
+        .unwrap_or_else(|e| panic!("Failed to create dictionary: {:?}", e));
+    let tokenizer = StatelessTokenizer::new(&dict);
 
     // tokenize and output results
     for line in reader.lines() {
-        let input = line.expect("Failed to reead line");
+        let input = line.expect("Failed to read line");
         for morpheme_list in tokenizer
             .tokenize_sentences(&input, mode, enable_debug)
             .expect("Failed to tokenize input")
@@ -136,36 +128,6 @@ fn main() {
     }
 }
 
-/// get dictionary bytes
-fn get_dictionary_bytes(system_dict: Option<&PathBuf>) -> Option<Cow<'static, [u8]>> {
-    let dictionary_path = {
-        cfg_if::cfg_if! {
-            if #[cfg(feature="bake_dictionary")] {
-                if let Some(dictionary_path) = system_dict {
-                    dictionary_path
-                } else {
-                    return Some(Cow::Borrowed(BAKED_DICTIONARY_BYTES));
-                }
-            } else {
-                if let Some(dictionary_path) = system_dict {
-                    dictionary_path
-                }else {
-                    return None;
-                }
-            }
-        }
-    };
-
-    let storage_buf = match dictionary_bytes_from_path(&dictionary_path) {
-        Ok(x) => x,
-        err => panic!(
-            "Failed to get dictionary bytes from file: {:?}\nError: {:?}",
-            &dictionary_path, &err
-        ),
-    };
-    Some(Cow::Owned(storage_buf))
-}
-
 /// Format and write morphemes into writer
 fn write_sentence(
     writer: &mut Box<dyn Write>,
diff --git a/sudachi/Cargo.toml b/sudachi/Cargo.toml
index 9fa26b0..1bb424d 100644
--- a/sudachi/Cargo.toml
+++ b/sudachi/Cargo.toml
@@ -18,6 +18,7 @@ nom = "7"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
+memmap2 = "0.5"
 
 [dev-dependencies]
 tempfile = "3"
diff --git a/sudachi/src/dic/dictionary.rs b/sudachi/src/dic/dictionary.rs
new file mode 100644
index 0000000..0160eeb
--- /dev/null
+++ b/sudachi/src/dic/dictionary.rs
@@ -0,0 +1,197 @@
+/*
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+use std::fs::File;
+use std::path::Path;
+
+use memmap2::Mmap;
+
+use crate::config::Config;
+use crate::config::ConfigError::MissingArgument;
+use crate::dic::grammar::Grammar;
+use crate::dic::lexicon_set::LexiconSet;
+use crate::dic::{DictionaryLoader, LoadedDictionary};
+use crate::error::SudachiError::ConfigError;
+use crate::error::{SudachiError, SudachiResult};
+use crate::plugin::input_text::InputTextPlugin;
+use crate::plugin::oov::OovProviderPlugin;
+use crate::plugin::path_rewrite::PathRewritePlugin;
+use crate::plugin::Plugins;
+use crate::stateless_tokeniser::{DictionaryAccess, StatelessTokenizer};
+
+// Mmap stores file handle, so file field may not be needed
+// but let it be here for the time being
+struct FileMapping {
+    file: File,
+    mapping: Mmap,
+}
+
+impl FileMapping {
+    // super unsafe, but we don't leak static lifetime outside of dictionary
+    // this is used only for construction of dictionary parts
+    unsafe fn as_static_slice(&self) -> &'static [u8] {
+        let slice: &[u8] = self.mapping.as_ref();
+        std::mem::transmute(slice)
+    }
+}
+
+enum StorageBackend {
+    FileSystem {
+        system: FileMapping,
+        user: Vec<FileMapping>,
+    },
+    InMemory {},
+}
+
+impl StorageBackend {
+    pub(crate) fn user_dicts_iter(&self) -> impl Iterator<Item = &'static [u8]> + '_ {
+        match self {
+            StorageBackend::FileSystem { user: u, .. } => {
+                u.iter().map(|m| unsafe { m.as_static_slice() })
+            }
+            _ => todo!(),
+        }
+    }
+}
+
+impl Drop for StorageBackend {
+    fn drop(&mut self) {
+        match self {
+            StorageBackend::FileSystem { user: u, .. } => {
+                u.clear();
+            }
+            _ => todo!(),
+        }
+    }
+}
+
+pub struct JapaneseDictionary {
+    backend: StorageBackend,
+    plugins: Plugins,
+    //'static is a a lie, lifetime is the same with StorageBackend
+    _grammar: Grammar<'static>,
+    //'static is a a lie, lifetime is the same with StorageBackend
+    _lexicon: LexiconSet<'static>,
+}
+
+fn map_file(path: &Path) -> SudachiResult<FileMapping> {
+    let file = File::open(path)?;
+    let mapping = unsafe { Mmap::map(&file) }?;
+    Ok(FileMapping { file, mapping })
+}
+
+fn load_system_dic(cfg: &Config) -> SudachiResult<FileMapping> {
+    match &cfg.system_dict {
+        Some(p) => map_file(p),
+        None => return Err(ConfigError(MissingArgument(String::from("system_dict")))),
+    }
+}
+
+fn load_user_dics(cfg: &Config) -> SudachiResult<Vec<FileMapping>> {
+    cfg.user_dicts.iter().map(|p| map_file(p)).collect()
+}
+
+impl JapaneseDictionary {
+    pub fn from_cfg(cfg: &Config) -> SudachiResult<JapaneseDictionary> {
+        let sys_dic = load_system_dic(cfg)?;
+        let user_dics = load_user_dics(cfg)?;
+
+        let mut basic_dict = LoadedDictionary::from_system_dictionary(
+            unsafe { sys_dic.as_static_slice() },
+            &cfg.character_definition_file,
+        )?;
+
+        let mut plugins = Plugins::new();
+
+        plugins.load(cfg, &basic_dict.grammar)?;
+
+        if plugins.oov.is_empty() {
+            return Err(SudachiError::NoOOVPluginProvided);
+        }
+
+        for p in plugins.connect_cost.plugins() {
+            p.edit(&mut basic_dict.grammar);
+        }
+
+        let mut dic = JapaneseDictionary {
+            backend: StorageBackend::FileSystem {
+                system: sys_dic,
+                user: user_dics,
+            },
+            plugins: plugins,
+            _grammar: basic_dict.grammar,
+            _lexicon: basic_dict.lexicon_set,
+        };
+
+        // this Vec is needed to prevent double borrowing of dic
+        let user_dicts: Vec<_> = dic.backend.user_dicts_iter().collect();
+        for udic in user_dicts {
+            dic = dic.merge_user_dictionary(udic)?;
+        }
+
+        Ok(dic)
+    }
+
+    /// Returns grammar with the correct lifetime
+    pub fn grammar<'a>(&'a self) -> &Grammar<'a> {
+        &self._grammar
+    }
+
+    /// Returns lexicon with the correct lifetime
+    pub fn lexicon<'a>(&'a self) -> &LexiconSet<'a> {
+        &self._lexicon
+    }
+
+    fn merge_user_dictionary(mut self, dictionary_bytes: &'static [u8]) -> SudachiResult<Self> {
+        let user_dict = DictionaryLoader::read_user_dictionary(dictionary_bytes)?;
+
+        // we need to update lexicon first, since it needs the current number of pos
+        let mut user_lexicon = user_dict.lexicon;
+        let tokenizer = StatelessTokenizer::new(&self);
+        user_lexicon.update_cost(&tokenizer)?;
+
+        self._lexicon
+            .append(user_lexicon, self._grammar.pos_list.len())?;
+
+        if let Some(g) = user_dict.grammar {
+            self._grammar.merge(g);
+        }
+
+        Ok(self)
+    }
+}
+
+impl DictionaryAccess for JapaneseDictionary {
+    fn grammar(&self) -> &Grammar<'_> {
+        self.grammar()
+    }
+
+    fn lexicon(&self) -> &LexiconSet<'_> {
+        self.lexicon()
+    }
+
+    fn input_text_plugins(&self) -> &[Box<dyn InputTextPlugin + Sync>] {
+        self.plugins.input_text.plugins()
+    }
+
+    fn oov_provider_plugins(&self) -> &[Box<dyn OovProviderPlugin + Sync>] {
+        self.plugins.oov.plugins()
+    }
+
+    fn path_rewrite_plugins(&self) -> &[Box<dyn PathRewritePlugin + Sync>] {
+        self.plugins.path_rewrite.plugins()
+    }
+}
diff --git a/sudachi/src/dic/lexicon/mod.rs b/sudachi/src/dic/lexicon/mod.rs
index ab80b21..4ed3fdb 100644
--- a/sudachi/src/dic/lexicon/mod.rs
+++ b/sudachi/src/dic/lexicon/mod.rs
@@ -103,7 +103,7 @@ impl<'a> Lexicon<'a> {
     }
 
     /// update word_param cost based on current tokenizer
-    pub fn update_cost(&mut self, tokenizer: &Tokenizer) -> SudachiResult<()> {
+    pub fn update_cost<T: Tokenize>(&mut self, tokenizer: &T) -> SudachiResult<()> {
         for wid in 0..self.word_params.size() as u32 {
             if self.word_params.get_cost(wid)? != std::i16::MIN {
                 continue;
diff --git a/sudachi/src/dic/mod.rs b/sudachi/src/dic/mod.rs
index 07c7582..78d6dff 100644
--- a/sudachi/src/dic/mod.rs
+++ b/sudachi/src/dic/mod.rs
@@ -27,24 +27,25 @@ use lexicon_set::LexiconSet;
 
 pub mod category_type;
 pub mod character_category;
+pub mod dictionary;
 pub mod grammar;
 pub mod header;
 pub mod lexicon;
 pub mod lexicon_set;
 
 /// A dictionary consists of one system_dict and zero or more user_dicts
-pub struct Dictionary<'a> {
+pub struct LoadedDictionary<'a> {
     pub grammar: Grammar<'a>,
     pub lexicon_set: LexiconSet<'a>,
 }
 
-impl<'a> Dictionary<'a> {
+impl<'a> LoadedDictionary<'a> {
     /// Creates a system dictionary from bytes, and load a character category from file
     pub fn from_system_dictionary(
         dictionary_bytes: &'a [u8],
         character_category_file: &PathBuf,
-    ) -> SudachiResult<Dictionary<'a>> {
-        let system_dict = BinaryDictionary::from_system_dictionary(dictionary_bytes)?;
+    ) -> SudachiResult<LoadedDictionary<'a>> {
+        let system_dict = DictionaryLoader::read_system_dictionary(dictionary_bytes)?;
 
         let character_category = CharacterCategory::from_file(character_category_file)?;
         let mut grammar = system_dict
@@ -52,7 +53,7 @@ impl<'a> Dictionary<'a> {
             .ok_or(SudachiError::InvalidDictionaryGrammar)?;
         grammar.set_character_category(character_category);
 
-        Ok(Dictionary {
+        Ok(LoadedDictionary {
             grammar,
             lexicon_set: LexiconSet::new(system_dict.lexicon),
         })
@@ -60,15 +61,15 @@ impl<'a> Dictionary<'a> {
 }
 
 /// A single system or user dictionary
-pub struct BinaryDictionary<'a> {
+pub struct DictionaryLoader<'a> {
     pub header: Header,
     pub grammar: Option<Grammar<'a>>,
     pub lexicon: Lexicon<'a>,
 }
 
-impl<'a> BinaryDictionary<'a> {
+impl<'a> DictionaryLoader<'a> {
     /// Creates a binary dictionary from bytes
-    fn read_dictionary(dictionary_bytes: &[u8]) -> SudachiResult<BinaryDictionary> {
+    fn read_dictionary(dictionary_bytes: &[u8]) -> SudachiResult<DictionaryLoader> {
         let header = Header::new(&dictionary_bytes[..Header::STORAGE_SIZE])?;
         let mut offset = Header::STORAGE_SIZE;
 
@@ -82,7 +83,7 @@ impl<'a> BinaryDictionary<'a> {
 
         let lexicon = Lexicon::new(dictionary_bytes, offset, header.has_synonym_group_ids())?;
 
-        Ok(BinaryDictionary {
+        Ok(DictionaryLoader {
             header,
             grammar,
             lexicon,
@@ -92,7 +93,7 @@ impl<'a> BinaryDictionary<'a> {
     /// Creates a system binary dictionary from bytes
     ///
     /// Returns Err if header version is not match
-    pub fn from_system_dictionary(dictionary_bytes: &[u8]) -> SudachiResult<BinaryDictionary> {
+    pub fn read_system_dictionary(dictionary_bytes: &[u8]) -> SudachiResult<DictionaryLoader> {
         let dict = Self::read_dictionary(dictionary_bytes)?;
         match dict.header.version {
             header::HeaderVersion::SystemDict(_) => Ok(dict),
@@ -105,7 +106,7 @@ impl<'a> BinaryDictionary<'a> {
     /// Creates a user binary dictionary from bytes
     ///
     /// Returns Err if header version is not match
-    pub fn from_user_dictionary(dictionary_bytes: &[u8]) -> SudachiResult<BinaryDictionary> {
+    pub fn read_user_dictionary(dictionary_bytes: &[u8]) -> SudachiResult<DictionaryLoader> {
         let dict = Self::read_dictionary(dictionary_bytes)?;
         match dict.header.version {
             header::HeaderVersion::UserDict(_) => Ok(dict),
diff --git a/sudachi/src/input_text/mod.rs b/sudachi/src/input_text/mod.rs
index 61fcd18..a5181fe 100644
--- a/sudachi/src/input_text/mod.rs
+++ b/sudachi/src/input_text/mod.rs
@@ -52,7 +52,7 @@ mod tests {
         grammar.set_character_category(character_category);
         grammar
     }
-    fn set_up_builder<'a>(grammar: &'a Grammar) -> Utf8InputTextBuilder<'a> {
+    fn set_up_builder<'a>(grammar: &'a Grammar) -> Utf8InputTextBuilder<'a, 'static> {
         Utf8InputTextBuilder::new(TEXT, grammar)
     }
 
diff --git a/sudachi/src/input_text/utf8_input_text_builder.rs b/sudachi/src/input_text/utf8_input_text_builder.rs
index 82ce33a..05fc58a 100644
--- a/sudachi/src/input_text/utf8_input_text_builder.rs
+++ b/sudachi/src/input_text/utf8_input_text_builder.rs
@@ -23,10 +23,10 @@ use crate::dic::grammar::Grammar;
 /// Builder of Uuf8InputText
 ///
 /// This handles modifications of the original text
-pub struct Utf8InputTextBuilder<'a> {
+pub struct Utf8InputTextBuilder<'a, 'b> {
     grammar: &'a Grammar<'a>,
     /// The original text
-    pub original: &'a str,
+    pub original: &'b str,
     /// The text after modifications
     pub modified: String,
     /// The mapping from modified byte_idx to original byte_idx.
@@ -35,8 +35,8 @@ pub struct Utf8InputTextBuilder<'a> {
     modified_to_original: Vec<usize>,
 }
 
-impl<'a> Utf8InputTextBuilder<'a> {
-    pub fn new(original: &'a str, grammar: &'a Grammar) -> Utf8InputTextBuilder<'a> {
+impl<'a, 'b> Utf8InputTextBuilder<'a, 'b> {
+    pub fn new(original: &'b str, grammar: &'a Grammar) -> Utf8InputTextBuilder<'a, 'b> {
         let modified = String::from(original);
 
         let modified_to_original: Vec<usize> = vec![0]
@@ -61,7 +61,7 @@ impl<'a> Utf8InputTextBuilder<'a> {
     /// Builds a Utf8InputText
     ///
     /// Generated Utf8InputText has a reference to this builder thus fn replace cannot be used after this.
-    pub fn build(self) -> Utf8InputText<'a> {
+    pub fn build(self: Self) -> Utf8InputText<'b> {
         let byte_indexes: Vec<usize> = self
             .modified
             .chars()
@@ -88,7 +88,7 @@ impl<'a> Utf8InputTextBuilder<'a> {
     }
 }
 
-impl Utf8InputTextBuilder<'_> {
+impl Utf8InputTextBuilder<'_, '_> {
     /// Replaces a substring of the current text by given new string
     pub fn replace(&mut self, char_range: Range<usize>, str_: &str) {
         let Range { start, end } = self.char_range_to_byte_range(char_range);
diff --git a/sudachi/src/lib.rs b/sudachi/src/lib.rs
index a6a310d..8472e85 100644
--- a/sudachi/src/lib.rs
+++ b/sudachi/src/lib.rs
@@ -29,6 +29,7 @@ pub mod lattice;
 pub mod morpheme;
 pub mod plugin;
 pub mod sentence_detector;
+pub mod stateless_tokeniser;
 pub mod tokenizer;
 
 pub mod prelude {
@@ -36,6 +37,6 @@ pub mod prelude {
         error::SudachiError,
         error::SudachiResult,
         morpheme::Morpheme,
-        tokenizer::{dictionary_bytes_from_path, Mode, Tokenize, Tokenizer},
+        tokenizer::{Mode, Tokenize},
     };
 }
diff --git a/sudachi/src/plugin/connect_cost.rs b/sudachi/src/plugin/connect_cost.rs
index 28c1dda..9c03a2e 100644
--- a/sudachi/src/plugin/connect_cost.rs
+++ b/sudachi/src/plugin/connect_cost.rs
@@ -23,7 +23,7 @@ use crate::dic::grammar::Grammar;
 use crate::prelude::*;
 
 /// Trait of plugin to edit connection cost in the grammar
-pub trait EditConnectionCostPlugin {
+pub trait EditConnectionCostPlugin: Sync {
     /// Loads necessary information for the plugin
     fn set_up(&mut self, settings: &Value, config: &Config, grammar: &Grammar)
         -> SudachiResult<()>;
@@ -42,12 +42,12 @@ pub trait EditConnectionCostPlugin {
 macro_rules! declare_connect_cost_plugin {
     ($plugin_type:ty, $constructor:path) => {
         #[no_mangle]
-        pub extern "C" fn load_plugin() -> *mut EditConnectionCostPlugin {
+        pub extern "C" fn load_plugin() -> *mut (dyn EditConnectionCostPlugin + Sync) {
             // make sure the constructor is the correct type.
             let constructor: fn() -> $plugin_type = $constructor;
 
             let object = constructor();
-            let boxed: Box<EditConnectionCostPlugin> = Box::new(object);
+            let boxed: Box<dyn EditConnectionCostPlugin + Sync> = Box::new(object);
             Box::into_raw(boxed)
         }
     };
@@ -56,7 +56,7 @@ macro_rules! declare_connect_cost_plugin {
 /// Plugin manager to handle multiple plugins
 #[derive(Default)]
 pub struct EditConnectionCostPluginManager {
-    plugins: Vec<Box<dyn EditConnectionCostPlugin>>,
+    plugins: Vec<Box<dyn EditConnectionCostPlugin + Sync>>,
     libraries: Vec<Library>,
 }
 impl EditConnectionCostPluginManager {
@@ -67,8 +67,7 @@ impl EditConnectionCostPluginManager {
         config: &Config,
         grammar: &Grammar,
     ) -> SudachiResult<()> {
-        type PluginCreate = unsafe fn() -> *mut dyn EditConnectionCostPlugin;
-
+        type PluginCreate = unsafe fn() -> *mut (dyn EditConnectionCostPlugin + Sync);
         let lib = unsafe { Library::new(path) }?;
         let load_plugin: Symbol<PluginCreate> = unsafe { lib.get(b"load_plugin") }?;
         let mut plugin = unsafe { Box::from_raw(load_plugin()) };
@@ -79,7 +78,7 @@ impl EditConnectionCostPluginManager {
         Ok(())
     }
 
-    pub fn plugins(&self) -> &[Box<dyn EditConnectionCostPlugin>] {
+    pub fn plugins(&self) -> &[Box<dyn EditConnectionCostPlugin + Sync>] {
         &self.plugins
     }
 
diff --git a/sudachi/src/plugin/input_text.rs b/sudachi/src/plugin/input_text.rs
index f194e61..64e8496 100644
--- a/sudachi/src/plugin/input_text.rs
+++ b/sudachi/src/plugin/input_text.rs
@@ -24,7 +24,7 @@ use crate::input_text::Utf8InputTextBuilder;
 use crate::prelude::*;
 
 /// Trait of plugin to modify the input text before tokenization
-pub trait InputTextPlugin {
+pub trait InputTextPlugin: Sync {
     /// Loads necessary information for the plugin
     fn set_up(&mut self, settings: &Value, config: &Config, grammar: &Grammar)
         -> SudachiResult<()>;
@@ -50,7 +50,7 @@ macro_rules! declare_input_text_plugin {
             let constructor: fn() -> $plugin_type = $constructor;
 
             let object = constructor();
-            let boxed: Box<dyn InputTextPlugin + Sync> = Box::new(object);
+            let boxed: Box<dyn InputTextPlugin + Sync + Send> = Box::new(object);
             Box::into_raw(boxed)
         }
     };
@@ -62,6 +62,7 @@ pub struct InputTextPluginManager {
     plugins: Vec<Box<dyn InputTextPlugin + Sync>>,
     libraries: Vec<Library>,
 }
+
 impl InputTextPluginManager {
     pub fn load(
         &mut self,
@@ -90,6 +91,7 @@ impl InputTextPluginManager {
         self.plugins.is_empty()
     }
 }
+
 impl Drop for InputTextPluginManager {
     fn drop(&mut self) {
         // Plugin drop must be called before Library drop.
diff --git a/sudachi/src/plugin/mod.rs b/sudachi/src/plugin/mod.rs
index 291f64a..ec39e2e 100644
--- a/sudachi/src/plugin/mod.rs
+++ b/sudachi/src/plugin/mod.rs
@@ -21,9 +21,15 @@ use thiserror::Error;
 
 use crate::config::ConfigError::FileNotFound;
 use crate::config::{Config, ConfigError};
+use crate::dic::grammar::Grammar;
+use crate::plugin::connect_cost::{
+    get_edit_connection_cost_plugins, EditConnectionCostPluginManager,
+};
+use crate::plugin::input_text::{get_input_text_plugins, InputTextPluginManager};
+use crate::plugin::oov::{get_oov_plugins, OovProviderPluginManager};
+use crate::plugin::path_rewrite::{get_path_rewrite_plugins, PathRewritePluginManager};
 use crate::prelude::*;
 use std::ffi::OsStr;
-use thiserror::private::PathAsDisplay;
 
 pub mod connect_cost;
 pub mod input_text;
@@ -109,7 +115,33 @@ pub fn get_plugin_path(plugin_config: &Value, config: &Config) -> SudachiResult<
     }
     Err(SudachiError::ConfigError(FileNotFound(format!(
         "Failed to find library, tried: {} and {}",
-        lib_path.as_display(),
-        fixed_path.as_display()
+        lib_path.display(),
+        fixed_path.display()
     ))))
 }
+
+pub(crate) struct Plugins {
+    pub(crate) connect_cost: EditConnectionCostPluginManager,
+    pub(crate) input_text: InputTextPluginManager,
+    pub(crate) oov: OovProviderPluginManager,
+    pub(crate) path_rewrite: PathRewritePluginManager,
+}
+
+impl Plugins {
+    pub(crate) fn new() -> Plugins {
+        return Plugins {
+            connect_cost: Default::default(),
+            input_text: Default::default(),
+            oov: Default::default(),
+            path_rewrite: Default::default(),
+        };
+    }
+
+    pub(crate) fn load(&mut self, cfg: &Config, grammar: &Grammar) -> SudachiResult<()> {
+        self.connect_cost = get_edit_connection_cost_plugins(cfg, grammar)?;
+        self.input_text = get_input_text_plugins(cfg, grammar)?;
+        self.oov = get_oov_plugins(cfg, grammar)?;
+        self.path_rewrite = get_path_rewrite_plugins(cfg, grammar)?;
+        Ok(())
+    }
+}
diff --git a/sudachi/src/plugin/oov.rs b/sudachi/src/plugin/oov.rs
index 79c3b1a..f95a2f4 100644
--- a/sudachi/src/plugin/oov.rs
+++ b/sudachi/src/plugin/oov.rs
@@ -25,7 +25,7 @@ use crate::lattice::node::Node;
 use crate::prelude::*;
 
 /// Trait of plugin to provide oov node during tokenization
-pub trait OovProviderPlugin {
+pub trait OovProviderPlugin: Sync {
     /// Loads necessary information for the plugin
     fn set_up(&mut self, settings: &Value, config: &Config, grammar: &Grammar)
         -> SudachiResult<()>;
@@ -109,6 +109,7 @@ impl OovProviderPluginManager {
         self.plugins.is_empty()
     }
 }
+
 impl Drop for OovProviderPluginManager {
     fn drop(&mut self) {
         // Plugin drop must be called before Library drop.
diff --git a/sudachi/src/plugin/path_rewrite.rs b/sudachi/src/plugin/path_rewrite.rs
index f81e8dd..7e913fb 100644
--- a/sudachi/src/plugin/path_rewrite.rs
+++ b/sudachi/src/plugin/path_rewrite.rs
@@ -26,7 +26,7 @@ use crate::lattice::{node::Node, Lattice};
 use crate::prelude::*;
 
 /// Trait of plugin to rewrite the path from lattice
-pub trait PathRewritePlugin {
+pub trait PathRewritePlugin: Sync {
     /// Loads necessary information for the plugin
     fn set_up(&mut self, settings: &Value, config: &Config, grammar: &Grammar)
         -> SudachiResult<()>;
diff --git a/sudachi/src/stateless_tokeniser.rs b/sudachi/src/stateless_tokeniser.rs
new file mode 100644
index 0000000..07add8e
--- /dev/null
+++ b/sudachi/src/stateless_tokeniser.rs
@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2021 Works Applications Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use std::ops::Deref;
+
+use crate::dic::category_type::CategoryType;
+use crate::dic::grammar::Grammar;
+use crate::dic::lexicon_set::LexiconSet;
+use crate::error::{SudachiError, SudachiResult};
+use crate::input_text::{Utf8InputText, Utf8InputTextBuilder};
+use crate::lattice::node::Node;
+use crate::lattice::Lattice;
+use crate::morpheme::Morpheme;
+use crate::plugin::input_text::InputTextPlugin;
+use crate::plugin::oov::OovProviderPlugin;
+use crate::plugin::path_rewrite::PathRewritePlugin;
+use crate::prelude::{Mode, Tokenize};
+use crate::sentence_detector::{NonBreakChecker, SentenceDetector};
+
+/// Provides access to dictionary data
+pub trait DictionaryAccess {
+    fn grammar(&self) -> &Grammar<'_>;
+    fn lexicon(&self) -> &LexiconSet<'_>;
+    fn input_text_plugins(&self) -> &[Box<dyn InputTextPlugin + Sync>];
+    fn oov_provider_plugins(&self) -> &[Box<dyn OovProviderPlugin + Sync>];
+    fn path_rewrite_plugins(&self) -> &[Box<dyn PathRewritePlugin + Sync>];
+}
+
+/// Implementation of a Tokenizer which does not have tokenization state.
+///
+/// This is a wrapper which is generic over dictionary pointers.
+/// Usable where dictionary is a struct itself, &, &mut, Rc<.>, Arc<.>.
+pub struct StatelessTokenizer<T> {
+    dict: T,
+}
+
+impl<T> StatelessTokenizer<T>
+where
+    T: Deref,
+    <T as Deref>::Target: DictionaryAccess,
+{
+    pub fn new(dict: T) -> StatelessTokenizer<T> {
+        StatelessTokenizer { dict }
+    }
+
+    pub fn as_dict(&self) -> &<T as Deref>::Target {
+        return Deref::deref(&self.dict);
+    }
+}
+
+impl<T: Deref> Tokenize for StatelessTokenizer<T>
+where
+    <T as Deref>::Target: DictionaryAccess,
+{
+    fn tokenize(
+        &self,
+        input: &str,
+        mode: Mode,
+        enable_debug: bool,
+    ) -> SudachiResult<Vec<Morpheme>> {
+        let dict = Deref::deref(&self.dict);
+        let input_text = build_input_text(dict, input);
+        tokenize_input_text(dict, &input_text, mode, enable_debug)
+    }
+
+    fn tokenize_sentences(
+        &self,
+        input: &str,
+        mode: Mode,
+        enable_debug: bool,
+    ) -> SudachiResult<Vec<Vec<Morpheme>>> {
+        if input.is_empty() {
+            return Ok(vec![Vec::new()]);
+        }
+
+        let dict = Deref::deref(&self.dict);
+        let input = build_input_text(dict, input);
+        split_sentences(dict.lexicon(), &input)?
+            .iter()
+            .map(|s| tokenize_input_text(dict, s, mode, enable_debug))
+            .collect()
+    }
+}
+
+fn split_sentences<'a, 'b>(
+    lexicon: &'a LexiconSet,
+    input: &'b Utf8InputText<'a>,
+) -> SudachiResult<Vec<Utf8InputText<'b>>> {
+    let mut sentences = Vec::new();
+    let mut checker = NonBreakChecker::new(lexicon, input);
+    let detector = SentenceDetector::new();
+    loop {
+        let byte_length = detector
+            .get_eos(&input.modified[checker.bos..], Some(&checker))?
+            .abs() as usize; // detector mey return negative value
+        if byte_length == 0 {
+            break;
+        }
+        let mut eos = checker.bos + byte_length;
+        if eos < input.modified.len() {
+            eos = input.get_next_in_original(eos - 1);
+        }
+        sentences.push(input.slice(checker.bos, eos));
+        checker.bos = eos;
+    }
+    Ok(sentences)
+}
+
+fn build_input_text<'b, T: DictionaryAccess + ?Sized>(
+    dict: &T,
+    input: &'b str,
+) -> Utf8InputText<'b> {
+    let mut builder = Utf8InputTextBuilder::new(input, dict.grammar());
+
+    for plugin in dict.input_text_plugins() {
+        plugin.rewrite(&mut builder);
+    }
+    builder.build()
+}
+
+fn tokenize_input_text<'a, T: DictionaryAccess + ?Sized>(
+    dict: &'a T,
+    input: &Utf8InputText,
+    mode: Mode,
+    enable_debug: bool,
+) -> SudachiResult<Vec<Morpheme<'a>>> {
+    if enable_debug {
+        println!("=== Input dump:\n{}", input.modified);
+    }
+
+    let lattice = build_lattice(dict, input)?;
+    if enable_debug {
+        println!("=== Lattice dump:");
+        lattice.dump(dict.grammar(), dict.lexicon())?;
+    };
+
+    let mut path = lattice.get_best_path()?;
+    // fill word_info to safely unwrap during path_rewrite and split_path
+    for node in &mut path {
+        node.fill_word_info(dict.lexicon())?;
+    }
+    if enable_debug {
+        println!("=== Before Rewriting:");
+        dump_path(&path);
+    };
+
+    for plugin in dict.path_rewrite_plugins() {
+        path = plugin.rewrite(&input, path, &lattice)?;
+    }
+    let path = split_path(dict, path, mode)?;
+    if enable_debug {
+        println!("=== After Rewriting:");
+        dump_path(&path);
+        println!("===");
+    };
+
+    path.iter()
+        .map(|node| Morpheme::new(&node, &input, dict.grammar(), dict.lexicon()))
+        .collect::<SudachiResult<Vec<_>>>()
+}
+
+fn build_lattice<'a, 'b, T: DictionaryAccess + ?Sized>(
+    dict: &'a T,
+    input: &'b Utf8InputText,
+) -> SudachiResult<Lattice<'a>> {
+    let input_bytes = input.modified.as_bytes();
+    let mut lattice = Lattice::new(dict.grammar(), input_bytes.len());
+    for (i, _) in input_bytes.iter().enumerate() {
+        if !input.can_bow(i) || !lattice.has_previous_node(i) {
+            continue;
+        }
+
+        let mut has_word = false;
+        for (word_id, end) in dict.lexicon().lookup(&input_bytes, i)? {
+            if (end < input_bytes.len()) && !input.can_bow(end) {
+                continue;
+            }
+            has_word = true;
+            let (left_id, right_id, cost) = dict.lexicon().get_word_param(word_id)?;
+            let node = Node::new(left_id, right_id, cost, word_id);
+            lattice.insert(i, end, node)?;
+        }
+
+        // OOV
+        if !input
+            .get_char_category_types(i)
+            .contains(&CategoryType::NOOOVBOW)
+        {
+            for oov_provider in dict.oov_provider_plugins() {
+                for node in oov_provider.get_oov(&input, i, has_word)? {
+                    has_word = true;
+                    lattice.insert(node.begin, node.end, node)?;
+                }
+            }
+        }
+        if !has_word {
+            // use last oov_provider as default
+            for node in dict
+                .oov_provider_plugins()
+                .last()
+                .unwrap()
+                .get_oov(&input, i, has_word)?
+            {
+                has_word = true;
+                lattice.insert(node.begin, node.end, node)?;
+            }
+        }
+
+        if !has_word {
+            panic!("no morpheme found at {}", i);
+        }
+    }
+    lattice.connect_eos_node()?;
+
+    Ok(lattice)
+}
+
+fn split_path<T: DictionaryAccess + ?Sized>(
+    dict: &T,
+    path: Vec<Node>,
+    mode: Mode,
+) -> SudachiResult<Vec<Node>> {
+    if mode == Mode::C {
+        return Ok(path);
+    }
+
+    let mut new_path = Vec::with_capacity(path.len());
+    for node in path {
+        let word_info = node
+            .word_info
+            .as_ref()
+            .ok_or(SudachiError::MissingWordInfo)?;
+        let word_ids = match mode {
+            Mode::A => &word_info.a_unit_split,
+            Mode::B => &word_info.b_unit_split,
+            _ => unreachable!(),
+        };
+
+        if word_ids.len() <= 1 {
+            new_path.push(node);
+        } else {
+            let mut offset = node.begin;
+            for wid in word_ids {
+                let mut n = Node::new(0, 0, 0, *wid);
+                n.fill_word_info(dict.lexicon())?;
+                let length = n
+                    .word_info
+                    .as_ref()
+                    .ok_or(SudachiError::MissingWordInfo)?
+                    .head_word_length as usize;
+                n.set_range(offset, offset + length);
+                new_path.push(n);
+
+                offset += length;
+            }
+        }
+    }
+
+    new_path.shrink_to_fit();
+    Ok(new_path)
+}
+
+fn dump_path(path: &Vec<Node>) {
+    for (i, node) in (&path).iter().enumerate() {
+        println!("{}: {}", i, node);
+    }
+}
diff --git a/sudachi/src/tokenizer.rs b/sudachi/src/tokenizer.rs
index beb41fd..07b86e8 100644
--- a/sudachi/src/tokenizer.rs
+++ b/sudachi/src/tokenizer.rs
@@ -14,26 +14,10 @@
  * limitations under the License.
  */
 
-use std::fs::{self, File};
-use std::io::Read;
-use std::path::Path;
 use std::str::FromStr;
 
-use crate::config::Config;
-use crate::dic::category_type::CategoryType;
-use crate::dic::grammar::Grammar;
-use crate::dic::lexicon_set::LexiconSet;
-use crate::dic::{BinaryDictionary, Dictionary};
-use crate::input_text::{Utf8InputText, Utf8InputTextBuilder};
-use crate::lattice::node::Node;
-use crate::lattice::Lattice;
 use crate::morpheme::Morpheme;
-use crate::plugin::connect_cost;
-use crate::plugin::input_text::{self, InputTextPluginManager};
-use crate::plugin::oov::{self, OovProviderPluginManager};
-use crate::plugin::path_rewrite::{self, PathRewritePluginManager};
 use crate::prelude::*;
-use crate::sentence_detector::{NonBreakChecker, SentenceDetector};
 
 /// Able to tokenize Japanese text
 pub trait Tokenize {
@@ -50,15 +34,6 @@ pub trait Tokenize {
     ) -> SudachiResult<Vec<Vec<Morpheme>>>;
 }
 
-/// Tokenizes Japanese text
-pub struct Tokenizer<'a> {
-    pub grammar: Grammar<'a>,
-    pub lexicon: LexiconSet<'a>,
-    input_text_plugins: InputTextPluginManager,
-    oov_provider_plugins: OovProviderPluginManager,
-    path_rewrite_plugins: PathRewritePluginManager,
-}
-
 /// Unit to split text
 ///
 /// Some examples:
@@ -106,279 +81,3 @@ impl FromStr for Mode {
         }
     }
 }
-
-/// Return bytes of a `dictionary_path`
-pub fn dictionary_bytes_from_path<P: AsRef<Path>>(dictionary_path: P) -> SudachiResult<Vec<u8>> {
-    let dictionary_path = dictionary_path.as_ref();
-    let dictionary_stat = fs::metadata(&dictionary_path)?;
-    let mut dictionary_file = File::open(dictionary_path)?;
-    let mut dictionary_bytes = Vec::with_capacity(dictionary_stat.len() as usize);
-    dictionary_file.read_to_end(&mut dictionary_bytes)?;
-
-    Ok(dictionary_bytes)
-}
-
-impl<'a> Tokenizer<'a> {
-    /// Create `Tokenizer` from the raw bytes of a Sudachi dictionary.
-    pub fn from_dictionary_bytes(
-        dictionary_bytes: &'a [u8],
-        user_dictionary_bytes: &'a Vec<Box<[u8]>>,
-        config: &Config,
-    ) -> SudachiResult<Tokenizer<'a>> {
-        let dictionary = Dictionary::from_system_dictionary(
-            dictionary_bytes,
-            &config.character_definition_file,
-        )?;
-        let mut grammar = dictionary.grammar;
-        let lexicon = dictionary.lexicon_set;
-
-        let edit_connection_cost_plugins =
-            connect_cost::get_edit_connection_cost_plugins(&config, &grammar)?;
-        for plugin in edit_connection_cost_plugins.plugins() {
-            plugin.edit(&mut grammar);
-        }
-
-        let input_text_plugins = input_text::get_input_text_plugins(config, &grammar)?;
-        let oov_provider_plugins = oov::get_oov_plugins(config, &grammar)?;
-        if oov_provider_plugins.is_empty() {
-            return Err(SudachiError::NoOOVPluginProvided);
-        }
-        let path_rewrite_plugins = path_rewrite::get_path_rewrite_plugins(config, &grammar)?;
-
-        let mut tokenizer = Tokenizer {
-            grammar,
-            lexicon,
-            input_text_plugins,
-            oov_provider_plugins,
-            path_rewrite_plugins,
-        };
-
-        for user_dict in user_dictionary_bytes {
-            tokenizer.merge_user_dictionary(user_dict)?;
-        }
-
-        Ok(tokenizer)
-    }
-
-    fn merge_user_dictionary(&mut self, dictionary_bytes: &'a [u8]) -> SudachiResult<()> {
-        let user_dict = BinaryDictionary::from_user_dictionary(dictionary_bytes)?;
-
-        // we need to update lexicon first, since it needs the current number of pos
-        let mut user_lexicon = user_dict.lexicon;
-        user_lexicon.update_cost(self)?;
-        self.lexicon
-            .append(user_lexicon, self.grammar.pos_list.len())?;
-
-        if let Some(g) = user_dict.grammar {
-            self.grammar.merge(g);
-        }
-
-        Ok(())
-    }
-
-    fn build_input_text(&'a self, input: &'a str) -> Utf8InputText<'a> {
-        let mut builder = Utf8InputTextBuilder::new(input, &self.grammar);
-
-        for plugin in self.input_text_plugins.plugins() {
-            plugin.rewrite(&mut builder);
-        }
-        builder.build()
-    }
-
-    fn split_sentences(&self, input: &'a Utf8InputText) -> SudachiResult<Vec<Utf8InputText>> {
-        let mut sentences = Vec::new();
-        let mut checker = NonBreakChecker::new(&self.lexicon, input);
-        let detector = SentenceDetector::new();
-        loop {
-            let byte_length = detector
-                .get_eos(&input.modified[checker.bos..], Some(&checker))?
-                .abs() as usize; // detector mey return negative value
-            if byte_length == 0 {
-                break;
-            }
-            let mut eos = checker.bos + byte_length;
-            if eos < input.modified.len() {
-                eos = input.get_next_in_original(eos - 1);
-            }
-            sentences.push(input.slice(checker.bos, eos));
-            checker.bos = eos;
-        }
-        Ok(sentences)
-    }
-}
-
-impl Tokenize for Tokenizer<'_> {
-    fn tokenize_sentences(
-        &self,
-        input: &str,
-        mode: Mode,
-        enable_debug: bool,
-    ) -> SudachiResult<Vec<Vec<Morpheme>>> {
-        if input.is_empty() {
-            return Ok(vec![Vec::new()]);
-        }
-
-        let input = self.build_input_text(input);
-        self.split_sentences(&input)?
-            .iter()
-            .map(|s| self.tokenize_input_text(s, mode, enable_debug))
-            .collect()
-    }
-
-    fn tokenize(
-        &self,
-        input: &str,
-        mode: Mode,
-        enable_debug: bool,
-    ) -> SudachiResult<Vec<Morpheme>> {
-        if input.is_empty() {
-            return Ok(Vec::new());
-        }
-        let input = self.build_input_text(input);
-        self.tokenize_input_text(&input, mode, enable_debug)
-    }
-}
-
-impl Tokenizer<'_> {
-    fn tokenize_input_text(
-        &self,
-        input: &Utf8InputText,
-        mode: Mode,
-        enable_debug: bool,
-    ) -> SudachiResult<Vec<Morpheme>> {
-        if enable_debug {
-            println!("=== Input dump:\n{}", input.modified);
-        }
-
-        let lattice = self.build_lattice(input)?;
-        if enable_debug {
-            println!("=== Lattice dump:");
-            lattice.dump(&self.grammar, &self.lexicon)?;
-        };
-
-        let mut path = lattice.get_best_path()?;
-        // fill word_info to safely unwrap during path_rewrite and split_path
-        for node in &mut path {
-            node.fill_word_info(&self.lexicon)?;
-        }
-        if enable_debug {
-            println!("=== Before Rewriting:");
-            Tokenizer::dump_path(&path);
-        };
-
-        for plugin in self.path_rewrite_plugins.plugins() {
-            path = plugin.rewrite(&input, path, &lattice)?;
-        }
-        let path = self.split_path(path, mode)?;
-        if enable_debug {
-            println!("=== After Rewriting:");
-            Tokenizer::dump_path(&path);
-            println!("===");
-        };
-
-        path.iter()
-            .map(|node| Morpheme::new(&node, &input, &self.grammar, &self.lexicon))
-            .collect::<SudachiResult<Vec<_>>>()
-    }
-
-    fn build_lattice(&self, input: &Utf8InputText) -> SudachiResult<Lattice> {
-        let input_bytes = input.modified.as_bytes();
-        let mut lattice = Lattice::new(&self.grammar, input_bytes.len());
-        for (i, _) in input_bytes.iter().enumerate() {
-            if !input.can_bow(i) || !lattice.has_previous_node(i) {
-                continue;
-            }
-
-            let mut has_word = false;
-            for (word_id, end) in self.lexicon.lookup(&input_bytes, i)? {
-                if (end < input_bytes.len()) && !input.can_bow(end) {
-                    continue;
-                }
-                has_word = true;
-                let (left_id, right_id, cost) = self.lexicon.get_word_param(word_id)?;
-                let node = Node::new(left_id, right_id, cost, word_id);
-                lattice.insert(i, end, node)?;
-            }
-
-            // OOV
-            if !input
-                .get_char_category_types(i)
-                .contains(&CategoryType::NOOOVBOW)
-            {
-                for oov_provider in self.oov_provider_plugins.plugins() {
-                    for node in oov_provider.get_oov(&input, i, has_word)? {
-                        has_word = true;
-                        lattice.insert(node.begin, node.end, node)?;
-                    }
-                }
-            }
-            if !has_word {
-                // use last oov_provider as default
-                for node in self
-                    .oov_provider_plugins
-                    .plugins()
-                    .last()
-                    .unwrap()
-                    .get_oov(&input, i, has_word)?
-                {
-                    has_word = true;
-                    lattice.insert(node.begin, node.end, node)?;
-                }
-            }
-
-            if !has_word {
-                panic!("no morpheme found at {}", i);
-            }
-        }
-        lattice.connect_eos_node()?;
-
-        Ok(lattice)
-    }
-
-    fn split_path(&self, path: Vec<Node>, mode: Mode) -> SudachiResult<Vec<Node>> {
-        if mode == Mode::C {
-            return Ok(path);
-        }
-
-        let mut new_path = Vec::with_capacity(path.len());
-        for node in path {
-            let word_info = node
-                .word_info
-                .as_ref()
-                .ok_or(SudachiError::MissingWordInfo)?;
-            let word_ids = match mode {
-                Mode::A => &word_info.a_unit_split,
-                Mode::B => &word_info.b_unit_split,
-                _ => unreachable!(),
-            };
-
-            if word_ids.len() <= 1 {
-                new_path.push(node);
-            } else {
-                let mut offset = node.begin;
-                for wid in word_ids {
-                    let mut n = Node::new(0, 0, 0, *wid);
-                    n.fill_word_info(&self.lexicon)?;
-                    let length = n
-                        .word_info
-                        .as_ref()
-                        .ok_or(SudachiError::MissingWordInfo)?
-                        .head_word_length as usize;
-                    n.set_range(offset, offset + length);
-                    new_path.push(n);
-
-                    offset += length;
-                }
-            }
-        }
-
-        new_path.shrink_to_fit();
-        Ok(new_path)
-    }
-
-    fn dump_path(path: &Vec<Node>) {
-        for (i, node) in (&path).iter().enumerate() {
-            println!("{}: {}", i, node);
-        }
-    }
-}


Test patch:
diff --git a/sudachi/tests/common/mod.rs b/sudachi/tests/common/mod.rs
index cadd9fc..47a30db 100644
--- a/sudachi/tests/common/mod.rs
+++ b/sudachi/tests/common/mod.rs
@@ -14,15 +14,31 @@
  * limitations under the License.
  */
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 extern crate sudachi;
+use self::sudachi::dic::dictionary::JapaneseDictionary;
+use self::sudachi::stateless_tokeniser::{StatelessTokenizer};
+use std::fs;
+use std::fs::File;
+use std::io::Read;
+use std::sync::Arc;
 use sudachi::config::Config;
 use sudachi::dic::{grammar::Grammar, header::Header, lexicon::Lexicon};
 use sudachi::prelude::*;
 
+pub fn dictionary_bytes_from_path<P: AsRef<Path>>(dictionary_path: P) -> SudachiResult<Vec<u8>> {
+    let dictionary_path = dictionary_path.as_ref();
+    let dictionary_stat = fs::metadata(&dictionary_path)?;
+    let mut dictionary_file = File::open(dictionary_path)?;
+    let mut dictionary_bytes = Vec::with_capacity(dictionary_stat.len() as usize);
+    dictionary_file.read_to_end(&mut dictionary_bytes)?;
+
+    Ok(dictionary_bytes)
+}
+
 lazy_static! {
-    static ref TEST_CONFIG: Config = {
+    pub static ref TEST_CONFIG: Config = {
         let test_config_path = "tests/resources/sudachi.json";
         let conf = Config::new(Some(PathBuf::from(test_config_path)), None, None)
             .expect("Failed to read config file for test");
@@ -57,7 +73,29 @@ lazy_static! {
         Lexicon::new(&DICTIONARY_BYTES, offset, HEADER.has_synonym_group_ids())
             .expect("Failed to read lexicon for tests")
     };
-    pub static ref TOKENIZER: Tokenizer<'static> =
-        Tokenizer::from_dictionary_bytes(&DICTIONARY_BYTES, &USER_DICTIONARY_BYTES, &TEST_CONFIG,)
-            .expect("Failed to create Tokenizer for tests");
 }
+
+pub struct TestTokenizer {
+    tok: StatelessTokenizer<Arc<JapaneseDictionary>>,
+}
+
+impl TestTokenizer {
+    pub fn new() -> TestTokenizer {
+        let dict = JapaneseDictionary::from_cfg(&TEST_CONFIG).expect("failed to make dictionary");
+        let tok = StatelessTokenizer::new(Arc::new(dict));
+        return TestTokenizer { tok };
+    }
+
+    pub fn tokenize(&self, data: &str, mode: Mode) -> Vec<Morpheme> {
+        let result = self.tok.tokenize(data, mode, false);
+        result.expect("tokenization failed")
+    }
+
+    pub fn dict(&self) -> &JapaneseDictionary {
+        &self.tok.as_dict()
+    }
+}
+
+// lazy_static! {
+//     pub static ref TEST_TOKENIZER: TestTokenizer = TestTokenizer::new();
+// }
diff --git a/sudachi/tests/dictionary.rs b/sudachi/tests/dictionary.rs
index 9950c43..5c1df0c 100644
--- a/sudachi/tests/dictionary.rs
+++ b/sudachi/tests/dictionary.rs
@@ -18,7 +18,7 @@
 extern crate lazy_static;
 
 mod common;
-use common::{GRAMMAR, TOKENIZER};
+use common::{TestTokenizer, GRAMMAR};
 
 #[test]
 fn get_part_of_speech_size() {
@@ -26,7 +26,8 @@ fn get_part_of_speech_size() {
     assert_eq!(8, GRAMMAR.pos_list.len());
 
     // user test dict contains another pos
-    assert_eq!(9, TOKENIZER.grammar.pos_list.len());
+    let tokenizer = TestTokenizer::new();
+    assert_eq!(9, tokenizer.dict().grammar().pos_list.len());
 }
 
 #[test]
diff --git a/sudachi/tests/sentence_detector.rs b/sudachi/tests/sentence_detector.rs
index 99d3cdf..7c91711 100644
--- a/sudachi/tests/sentence_detector.rs
+++ b/sudachi/tests/sentence_detector.rs
@@ -18,7 +18,8 @@
 extern crate lazy_static;
 
 mod common;
-use common::{GRAMMAR, TOKENIZER};
+use common::TestTokenizer;
+use common::GRAMMAR;
 
 use sudachi::input_text::Utf8InputTextBuilder;
 use sudachi::sentence_detector::{NonBreakChecker, SentenceDetector};
@@ -28,7 +29,9 @@ fn get_eos_with_non_break_checker() {
     let text = "ばな。なです。";
     let builder = Utf8InputTextBuilder::new(&text, &GRAMMAR);
     let input = builder.build();
-    let checker = NonBreakChecker::new(&TOKENIZER.lexicon, &input);
+    let tokenizer = TestTokenizer::new();
+    let lexicon = tokenizer.dict().lexicon();
+    let checker = NonBreakChecker::new(lexicon, &input);
 
     let sd = SentenceDetector::new();
     assert_eq!(sd.get_eos(text, Some(&checker)).unwrap(), 21);
diff --git a/sudachi/tests/tokenizer.rs b/sudachi/tests/stateless_tokenizer.rs
similarity index 51%
rename from sudachi/tests/tokenizer.rs
rename to sudachi/tests/stateless_tokenizer.rs
index d57f96e..6f7b813 100644
--- a/sudachi/tests/tokenizer.rs
+++ b/sudachi/tests/stateless_tokenizer.rs
@@ -1,52 +1,45 @@
 /*
- * Copyright (c) 2021 Works Applications Co., Ltd.
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 
 #[macro_use]
 extern crate lazy_static;
-
 extern crate sudachi;
-use sudachi::prelude::*;
 
-mod common;
-use common::TOKENIZER;
+use sudachi::tokenizer::Mode;
 
-/// Get morpheme list from text
-fn tokenize(text: &str, mode: Mode) -> Vec<Morpheme> {
-    TOKENIZER
-        .tokenize(text, mode, false)
-        .expect("Failed to get tokens")
-}
+mod common;
+use crate::common::TestTokenizer;
 
 #[test]
 fn tokenize_small_katakana_only() {
-    let ms = tokenize("ァ", Mode::C);
+    let tok = TestTokenizer::new();
+    let ms = tok.tokenize("ァ", Mode::C);
     assert_eq!(1, ms.len());
 }
 
-// skip part_of_speech since Morpheme have pos field directly
-
 #[test]
 fn get_word_id() {
-    let ms = tokenize("京都", Mode::C);
+    let tok = TestTokenizer::new();
+    let ms = tok.tokenize("京都", Mode::C);
     assert_eq!(1, ms.len());
     let pos = ms[0].pos().expect("failed to get pos");
     assert_eq!(["名詞", "固有名詞", "地名", "一般", "*", "*"], &pos[..]);
 
     // we do not have word_id field in Morpheme and skip testing.
-    let ms = tokenize("ぴらる", Mode::C);
+    let ms = tok.tokenize("ぴらる", Mode::C);
     assert_eq!(1, ms.len());
     let pos = ms[0].pos().expect("failed to get pos");
     assert_eq!(["名詞", "普通名詞", "一般", "*", "*", "*"], &pos[..]);
@@ -54,44 +47,48 @@ fn get_word_id() {
 
 #[test]
 fn get_dictionary_id() {
-    let ms = tokenize("京都", Mode::C);
+    let tok = TestTokenizer::new();
+    let ms = tok.tokenize("京都", Mode::C);
     assert_eq!(1, ms.len());
     assert_eq!(0, ms[0].dictionary_id);
 
-    let ms = tokenize("ぴらる", Mode::C);
+    let ms = tok.tokenize("ぴらる", Mode::C);
     assert_eq!(1, ms.len());
     assert_eq!(1, ms[0].dictionary_id);
 
-    let ms = tokenize("京", Mode::C);
+    let ms = tok.tokenize("京", Mode::C);
     assert_eq!(1, ms.len());
     assert!(ms[0].dictionary_id < 0);
 }
 
 #[test]
 fn get_synonym_group_id() {
-    let ms = tokenize("京都", Mode::C);
+    let tok = TestTokenizer::new();
+    let ms = tok.tokenize("京都", Mode::C);
     assert_eq!(1, ms.len());
     assert_eq!([1, 5], ms[0].word_info.synonym_group_ids.as_slice());
 
-    let ms = tokenize("ぴらる", Mode::C);
+    let ms = tok.tokenize("ぴらる", Mode::C);
     assert_eq!(1, ms.len());
     assert!(ms[0].word_info.synonym_group_ids.is_empty());
 
-    let ms = tokenize("東京府", Mode::C);
+    let ms = tok.tokenize("東京府", Mode::C);
     assert_eq!(1, ms.len());
     assert_eq!([1, 3], ms[0].word_info.synonym_group_ids.as_slice());
 }
 
 #[test]
 fn tokenize_kanji_alphabet_word() {
-    assert_eq!(1, tokenize("特a", Mode::C).len());
-    assert_eq!(1, tokenize("ab", Mode::C).len());
-    assert_eq!(2, tokenize("特ab", Mode::C).len());
+    let tok = TestTokenizer::new();
+    assert_eq!(1, tok.tokenize("特a", Mode::C).len());
+    assert_eq!(1, tok.tokenize("ab", Mode::C).len());
+    assert_eq!(2, tok.tokenize("特ab", Mode::C).len());
 }
 
 #[test]
 fn tokenize_with_dots() {
-    let ms = tokenize("京都…", Mode::C);
+    let tok = TestTokenizer::new();
+    let ms = tok.tokenize("京都…", Mode::C);
     assert_eq!(4, ms.len());
     assert_eq!("…", ms[1].surface());
     assert_eq!(".", ms[1].normalized_form());
@@ -103,11 +100,12 @@ fn tokenize_with_dots() {
 
 #[test]
 fn tokenizer_morpheme_split() {
-    let ms = tokenize("東京都", Mode::C);
+    let tok = TestTokenizer::new();
+    let ms = tok.tokenize("東京都", Mode::C);
     assert_eq!(1, ms.len());
     assert_eq!("東京都", ms[0].surface());
 
-    let ms = tokenize("東京都", Mode::A);
+    let ms = tok.tokenize("東京都", Mode::A);
     assert_eq!(2, ms.len());
     assert_eq!("東京", ms[0].surface());
     assert_eq!("都", ms[1].surface());