Repository: WorksApplications/sudachi.rs (rust)

Issue:
pyo3_runtime.PanicException occurs in Morpheme.surface() after calling Morpheme.split()
I found an input pattern which causes exception in `sudachipy==0.6.0`.
The reproducing code below is the abstract of the Japanese tokenizer of spaCy v3.2.0.

```python
from sudachipy import dictionary, tokenizer

def get_dtokens(tokenizer, sudachipy_tokens, need_sub_tokens):
    sub_tokens_list = get_sub_tokens(tokenizer, sudachipy_tokens) if need_sub_tokens else None
    dtokens = [
        (
            t.surface(),
            t.part_of_speech()[:4],
            t.part_of_speech()[4:],
            t.dictionary_form(),
            t.normalized_form(),
            t.reading_form(),
            sub_tokens_list[idx] if need_sub_tokens else None,
        ) for idx, t in enumerate(sudachipy_tokens) if len(t.surface()) > 0
    ]
    return dtokens

def get_sub_tokens(tokenizer, sudachipy_tokens):
    sub_tokens_list = []
    for token in sudachipy_tokens:
        sub_a = token.split(tokenizer.SplitMode.A)
        if len(sub_a) == 1:  # no sub tokens
            sub_tokens_list.append(None)
        else:
            sub_b = token.split(tokenizer.SplitMode.B)
            if len(sub_a) == len(sub_b):
                dtokens = get_dtokens(tokenizer, sub_a, False)
                sub_tokens_list.append([dtokens, dtokens])
            else:
                sub_tokens_list.append(
                    [
                        get_dtokens(tokenizer, sub_a, False),
                        get_dtokens(tokenizer, sub_b, False),
                    ]
                )
    return sub_tokens_list

tokenizer = dictionary.Dictionary().create(mode=tokenizer.Tokenizer.SplitMode.C)
sudachipy_tokens = tokenizer.tokenize("Ｔ社はｅコマース（電子商取引）を活用したリサイクル部品の取扱いを系列の部品販売店で平成１３年１０月より始めました。取り扱う部品は、ドア、フェンダー、グリル、バンパー、ランプ類などのＴ社の外装部品（「エ コロパーツ」）全１６品目と大手リサイクル部品流通事業社のＮグループ及びＢ社から供給を受ける国内全メーカーの外装・機能部品で、専用の中古部品ｅコマースサイトを開設し、自動車保有期間の長期化に伴う低価格修理の需要に応えることにしています。")
get_dtokens(tokenizer, sudachipy_tokens, True)
```
```
/home/matsuda/ginza/test.py:21: DeprecationWarning: API around this functionality will change. See github issue WorksApplications/SudachiPy#92 for more.
  sub_a = token.split(tokenizer.SplitMode.A)
/home/matsuda/ginza/test.py:25: DeprecationWarning: API around this functionality will change. See github issue WorksApplications/SudachiPy#92 for more.
  sub_b = token.split(tokenizer.SplitMode.B)
thread '<unnamed>' panicked at 'byte index 10 is not a char boundary; it is inside 'ｅ' (bytes 9..12) of `Ｔ社はｅコマース（電子商取引）を活用したリサイクル部品の取扱いを系列の部品販売店で平成１３年１０月より始めました。取り扱う部品は、ドア、フェンダー、グリル、バンパー、ラン`[...]', /github/workspace/sudachi/src/analysis/morpheme.rs:122:10
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
Traceback (most recent call last):
  File "/home/matsuda/ginza/test.py", line 40, in <module>
    get_dtokens(tokenizer, sudachipy_tokens, True)
  File "/home/matsuda/ginza/test.py", line 4, in get_dtokens
    sub_tokens_list = get_sub_tokens(tokenizer, sudachipy_tokens) if need_sub_tokens else None
  File "/home/matsuda/ginza/test.py", line 27, in get_sub_tokens
    dtokens = get_dtokens(tokenizer, sub_a, False)
  File "/home/matsuda/ginza/test.py", line 5, in get_dtokens
    dtokens = [
  File "/home/matsuda/ginza/test.py", line 14, in <listcomp>
    ) for idx, t in enumerate(sudachipy_tokens) if len(t.surface()) > 0
pyo3_runtime.PanicException: byte index 10 is not a char boundary; it is inside 'ｅ' (bytes 9..12) of `Ｔ社はｅコマース（電子商取引）を活用したリサイクル部品の取扱いを系列の部品販売店で平成１３年１０月より始めました。取り扱う部品は、ドア、フェンダー、グリル、バンパー、ラン`[...]
```

Pull request:
Correctly resolve node boundaries wrt node splitting
fixes #182 

As a side effect, MorphemeList now has an `Arc<InputBuffer>` and all stored indices in `Node`s now are in modified space, not in original space. Original indices are computed only when requested (so `InputBuffer` is needed there).

Patch:
diff --git a/python/py_src/sudachipy/sudachipy.pyi b/python/py_src/sudachipy/sudachipy.pyi
index 6af7bff..cab09b2 100644
--- a/python/py_src/sudachipy/sudachipy.pyi
+++ b/python/py_src/sudachipy/sudachipy.pyi
@@ -177,6 +177,11 @@ class Morpheme:
         """
         ...
 
+    def __len__(self) -> int:
+        """
+        Returns morpheme length in codepoints
+        """
+
 
 class MorphemeList:
     """
diff --git a/python/src/morpheme.rs b/python/src/morpheme.rs
index cfecdb7..dfb0141 100644
--- a/python/src/morpheme.rs
+++ b/python/src/morpheme.rs
@@ -164,14 +164,14 @@ impl PyMorpheme {
     #[pyo3(text_signature = "($self)")]
     fn begin(&self) -> usize {
         // call codepoint version
-        self.list.get_node(self.index).begin()
+        self.list.get_begin_cp(self.index)
     }
 
     /// Returns the end index of this in the input text
     #[pyo3(text_signature = "($self)")]
     fn end(&self) -> usize {
         // call codepoint version
-        self.list.get_node(self.index).end()
+        self.list.get_end_cp(self.index)
     }
 
     /// Returns the surface
@@ -269,4 +269,9 @@ impl PyMorpheme {
 
         Ok(self.list.get_word_info(self.index).clone().into())
     }
+
+    /// Returns morpheme length in codepoints
+    pub fn __len__(&self) -> usize {
+        self.end() - self.begin()
+    }
 }
diff --git a/sudachi/src/analysis/morpheme.rs b/sudachi/src/analysis/morpheme.rs
index 8291ad0..6dd2802 100644
--- a/sudachi/src/analysis/morpheme.rs
+++ b/sudachi/src/analysis/morpheme.rs
@@ -21,13 +21,15 @@ use crate::dic::grammar::Grammar;
 use crate::dic::lexicon::word_infos::WordInfo;
 use crate::dic::subset::InfoSubset;
 use crate::dic::word_id::WordId;
+use crate::input_text::{InputBuffer, InputTextIndex};
 use crate::prelude::*;
+use std::sync::Arc;
 
 /// A list of morphemes
 #[derive(Clone)]
 pub struct MorphemeList<T> {
     dict: T,
-    input_text: String,
+    input_text: Arc<InputBuffer>,
     path: Vec<ResultNode>,
     subset: InfoSubset,
 }
@@ -35,13 +37,13 @@ pub struct MorphemeList<T> {
 impl<T: DictionaryAccess> MorphemeList<T> {
     pub fn from_components(
         dict: T,
-        original: String,
+        input: InputBuffer,
         path: Vec<ResultNode>,
         subset: InfoSubset,
     ) -> SudachiResult<Self> {
         let list = Self {
             dict,
-            input_text: original,
+            input_text: Arc::new(input),
             path,
             subset,
         };
@@ -50,11 +52,13 @@ impl<T: DictionaryAccess> MorphemeList<T> {
 
     /// Returns an empty morpheme list.
     pub fn empty(dict: T) -> Self {
+        let mut ml = InputBuffer::new();
+        ml.start_build().unwrap(); // will not fail
         Self {
             dict,
-            input_text: String::new(),
-            path: Vec::new(),
-            subset: InfoSubset::default(),
+            input_text: Arc::new(ml),
+            path: Default::default(),
+            subset: Default::default(),
         }
     }
 
@@ -62,7 +66,9 @@ impl<T: DictionaryAccess> MorphemeList<T> {
         &mut self,
         analyzer: &mut StatefulTokenizer<U>,
     ) -> SudachiResult<()> {
-        analyzer.swap_result(&mut self.input_text, &mut self.path, &mut self.subset);
+        // make a copy of input buffer if something else exists
+        let itext = Arc::make_mut(&mut self.input_text);
+        analyzer.swap_result(itext, &mut self.path, &mut self.subset);
         Ok(())
     }
 
@@ -80,7 +86,8 @@ impl<T: DictionaryAccess + Clone> MorphemeList<T> {
         let path = if num_splits <= 1 {
             vec![node.clone()]
         } else {
-            node.split(mode, self.dict.lexicon(), self.subset).collect()
+            node.split(mode, self.dict.lexicon(), self.subset, &self.input_text)
+                .collect()
         };
 
         Ok(MorphemeList {
@@ -102,13 +109,13 @@ impl<T> MorphemeList<T> {
 
     pub fn surface(&self) -> &str {
         if self.len() == 0 {
-            return &self.input_text;
+            return self.input_text.original();
         }
 
         // can be a slice of another list
         let begin = self.path.first().unwrap().begin_bytes();
         let end = self.path.last().unwrap().end_bytes();
-        &self.input_text[begin..end]
+        self.input_text.orig_slice(begin..end)
     }
 
     pub fn get_node(&self, index: usize) -> &ResultNode {
@@ -116,17 +123,31 @@ impl<T> MorphemeList<T> {
     }
 
     pub fn get_begin(&self, index: usize) -> usize {
-        self.path[index].begin_bytes()
+        let mod_bytes = self.path[index].begin();
+        self.input_text.to_orig_byte_idx(mod_bytes)
     }
 
     pub fn get_end(&self, index: usize) -> usize {
-        self.path[index].end_bytes()
+        let mod_bytes = self.path[index].end();
+        self.input_text.to_orig_byte_idx(mod_bytes)
+    }
+
+    /// Returns begin codepoint index in the original sentence
+    pub fn get_begin_cp(&self, index: usize) -> usize {
+        let begin_cp = self.path[index].begin();
+        self.input_text.to_orig_char_idx(begin_cp)
+    }
+
+    /// Returns end codepoint index in the original sentence
+    pub fn get_end_cp(&self, index: usize) -> usize {
+        let end_cp = self.path[index].end();
+        self.input_text.to_orig_char_idx(end_cp)
     }
 
     /// Returns a substring of the original text which corresponds to the morpheme
     pub fn get_surface(&self, index: usize) -> &str {
         let node = &self.path[index];
-        &self.input_text[node.begin_bytes()..node.end_bytes()]
+        &self.input_text.orig_slice(node.bytes_range())
     }
 
     pub fn get_word_info(&self, index: usize) -> &WordInfo {
@@ -214,12 +235,12 @@ impl<T: DictionaryAccess + Clone> Morpheme<'_, T> {
 }
 
 impl<T> Morpheme<'_, T> {
-    /// Returns the begin index of morpheme in the original text
+    /// Returns the begin index in bytes of the morpheme in the original text
     pub fn begin(&self) -> usize {
         self.list.get_begin(self.index)
     }
 
-    /// Returns the end index of morpehme in the original text
+    /// Returns the end index in bytes of the morpheme in the original text
     pub fn end(&self) -> usize {
         self.list.get_end(self.index)
     }
diff --git a/sudachi/src/analysis/node.rs b/sudachi/src/analysis/node.rs
index 6f8933e..2e84008 100644
--- a/sudachi/src/analysis/node.rs
+++ b/sudachi/src/analysis/node.rs
@@ -23,6 +23,7 @@ use crate::dic::lexicon::word_infos::{WordInfo, WordInfoData};
 use crate::dic::lexicon_set::LexiconSet;
 use crate::dic::subset::InfoSubset;
 use crate::dic::word_id::WordId;
+use crate::input_text::InputBuffer;
 use crate::prelude::*;
 
 /// Accessor trait for right connection id
@@ -76,7 +77,8 @@ pub trait LatticeNode: RightId {
 
 #[derive(Clone)]
 /// Full lattice node, as the result of analysis.
-/// All indices (including inner) are in the original sentence space.
+/// All indices (including inner) are in the modified sentence space
+/// Indices are converted to original sentence space when user request them.
 pub struct ResultNode {
     inner: Node,
     total_cost: i32,
@@ -181,6 +183,7 @@ impl ResultNode {
         mode: Mode,
         lexicon: &'a LexiconSet<'a>,
         subset: InfoSubset,
+        text: &'a InputBuffer,
     ) -> NodeSplitIterator<'a> {
         let splits: &[WordId] = match mode {
             Mode::A => &self.word_info.a_unit_split(),
@@ -193,6 +196,7 @@ impl ResultNode {
             index: 0,
             lexicon,
             subset,
+            text,
             byte_offset: self.begin_bytes,
             byte_end: self.end_bytes,
             char_offset: self.begin() as u16,
@@ -223,6 +227,7 @@ pub struct NodeSplitIterator<'a> {
     lexicon: &'a LexiconSet<'a>,
     index: usize,
     subset: InfoSubset,
+    text: &'a InputBuffer,
     char_offset: u16,
     byte_offset: u16,
     char_end: u16,
@@ -252,11 +257,9 @@ impl Iterator for NodeSplitIterator<'_> {
         let (char_end, byte_end) = if idx + 1 == self.splits.len() {
             (self.char_end, self.byte_end)
         } else {
-            (
-                // it is possible that surface is not accessible here, fixup when InputBuffer is available
-                char_start + word_info.surface().chars().count() as u16,
-                byte_start + word_info.head_word_length() as u16,
-            )
+            let byte_end = byte_start as usize + word_info.head_word_length();
+            let char_end = self.text.ch_idx(byte_end);
+            (char_end as u16, byte_end as u16)
         };
 
         self.char_offset = char_end;
diff --git a/sudachi/src/analysis/stateful_tokenizer.rs b/sudachi/src/analysis/stateful_tokenizer.rs
index 61023e9..0147277 100644
--- a/sudachi/src/analysis/stateful_tokenizer.rs
+++ b/sudachi/src/analysis/stateful_tokenizer.rs
@@ -144,9 +144,7 @@ impl<D: DictionaryAccess> StatefulTokenizer<D> {
             path = plugin.rewrite(&self.input, path, &self.lattice)?;
         }
 
-        path = split_path(&self.dictionary, path, self.mode, self.subset)?;
-
-        self.translate_indices(&mut path);
+        path = split_path(&self.dictionary, path, self.mode, self.subset, &self.input)?;
 
         if debug {
             println!("=== After Rewriting:");
@@ -193,33 +191,14 @@ impl<D: DictionaryAccess> StatefulTokenizer<D> {
         Ok(path)
     }
 
-    /// Translate ResultNode indices from normalized data to original data
-    fn translate_indices(&self, path: &mut Vec<ResultNode>) {
-        let input = &self.input;
-        for elem in path {
-            let mut mod_char_begin = elem.begin();
-            let mut mod_char_end = elem.end();
-            if !self.subset.contains(InfoSubset::SURFACE) {
-                mod_char_begin = input.ch_idx(elem.begin_bytes());
-                mod_char_end = input.ch_idx(elem.end_bytes());
-            }
-            let char_begin = input.to_orig_char_idx(mod_char_begin);
-            let char_end = input.to_orig_char_idx(mod_char_end);
-            let byte_begin = input.to_orig_byte_idx(mod_char_begin);
-            let byte_end = input.to_orig_byte_idx(mod_char_end);
-            elem.set_char_range(char_begin as u16, char_end as u16);
-            elem.set_bytes_range(byte_begin as u16, byte_end as u16);
-        }
-    }
-
     /// Swap result data with the current analyzer
     pub fn swap_result(
         &mut self,
-        input: &mut String,
+        input: &mut InputBuffer,
         result: &mut Vec<ResultNode>,
         subset: &mut InfoSubset,
     ) {
-        self.input.swap_original(input);
+        std::mem::swap(&mut self.input, input);
         std::mem::swap(self.top_path.as_mut().unwrap(), result);
         *subset = self.subset;
     }
@@ -304,12 +283,9 @@ impl<D: DictionaryAccess> StatefulTokenizer<D> {
     pub fn into_morpheme_list(self) -> SudachiResult<MorphemeList<D>> {
         match self.top_path {
             None => Err(SudachiError::EosBosDisconnect),
-            Some(path) => MorphemeList::from_components(
-                self.dictionary,
-                self.input.into_original(),
-                path,
-                self.subset,
-            ),
+            Some(path) => {
+                MorphemeList::from_components(self.dictionary, self.input, path, self.subset)
+            }
         }
     }
 }
diff --git a/sudachi/src/analysis/stateless_tokenizer.rs b/sudachi/src/analysis/stateless_tokenizer.rs
index 1a08406..0bd54c6 100644
--- a/sudachi/src/analysis/stateless_tokenizer.rs
+++ b/sudachi/src/analysis/stateless_tokenizer.rs
@@ -22,6 +22,7 @@ use crate::dic::grammar::Grammar;
 use crate::dic::lexicon_set::LexiconSet;
 use crate::dic::subset::InfoSubset;
 use crate::error::SudachiResult;
+use crate::input_text::InputBuffer;
 use crate::plugin::input_text::InputTextPlugin;
 use crate::plugin::oov::OovProviderPlugin;
 use crate::plugin::path_rewrite::PathRewritePlugin;
@@ -112,6 +113,7 @@ pub(super) fn split_path<T: DictionaryAccess + ?Sized>(
     path: Vec<ResultNode>,
     mode: Mode,
     subset: InfoSubset,
+    input: &InputBuffer,
 ) -> SudachiResult<Vec<ResultNode>> {
     if mode == Mode::C {
         return Ok(path);
@@ -123,7 +125,7 @@ pub(super) fn split_path<T: DictionaryAccess + ?Sized>(
         if split_len <= 1 {
             new_path.push(node);
         } else {
-            new_path.extend(node.split(mode, dict.lexicon(), subset));
+            new_path.extend(node.split(mode, dict.lexicon(), subset, input));
         }
     }
 
diff --git a/sudachi/src/dic/dictionary.rs b/sudachi/src/dic/dictionary.rs
index 4e01c77..2d5ea7e 100644
--- a/sudachi/src/dic/dictionary.rs
+++ b/sudachi/src/dic/dictionary.rs
@@ -24,6 +24,7 @@ use crate::config::Config;
 use crate::config::ConfigError::MissingArgument;
 use crate::dic::grammar::Grammar;
 use crate::dic::lexicon_set::LexiconSet;
+use crate::dic::storage::{Storage, SudachiDicData};
 use crate::dic::{DictionaryLoader, LoadedDictionary};
 use crate::error::SudachiError::ConfigError;
 use crate::error::{SudachiError, SudachiResult};
@@ -32,56 +33,6 @@ use crate::plugin::oov::OovProviderPlugin;
 use crate::plugin::path_rewrite::PathRewritePlugin;
 use crate::plugin::Plugins;
 
-// Mmap stores file handle, so file field may not be needed
-// but let it be here for the time being
-struct FileMapping {
-    #[allow(unused)]
-    file: File,
-    mapping: Mmap,
-}
-
-impl FileMapping {
-    // super unsafe, but we don't leak static lifetime outside of dictionary
-    // this is used only for construction of dictionary parts
-    unsafe fn as_static_slice(&self) -> &'static [u8] {
-        let slice: &[u8] = self.mapping.as_ref();
-        std::mem::transmute(slice)
-    }
-}
-
-enum StorageBackend {
-    FileSystem {
-        #[allow(unused)] // This is for ownership, this is not used by code
-        system: FileMapping,
-        user: Vec<FileMapping>,
-    },
-    // Remove when fixing https://github.com/WorksApplications/sudachi.rs/issues/35
-    #[allow(unused)]
-    InMemory {},
-}
-
-impl StorageBackend {
-    pub(crate) fn user_dicts_iter(&self) -> impl Iterator<Item = &'static [u8]> + '_ {
-        match self {
-            StorageBackend::FileSystem { user: u, .. } => {
-                u.iter().map(|m| unsafe { m.as_static_slice() })
-            }
-            _ => panic!("Not implemented"),
-        }
-    }
-}
-
-impl Drop for StorageBackend {
-    fn drop(&mut self) {
-        match self {
-            StorageBackend::FileSystem { user: u, .. } => {
-                u.clear();
-            }
-            _ => panic!("Not implemented"),
-        }
-    }
-}
-
 // It is self-referential struct with 'static lifetime as a workaround
 // for the impossibility to specify the correct lifetime for
 // those fields. Accessor functions always provide the correct lifetime,
@@ -92,7 +43,7 @@ impl Drop for StorageBackend {
 // This structure is always read only after creation and is safe to share
 // between threads.
 pub struct JapaneseDictionary {
-    backend: StorageBackend,
+    storage: SudachiDicData,
     plugins: Plugins,
     //'static is a a lie, lifetime is the same with StorageBackend
     _grammar: Grammar<'static>,
@@ -100,30 +51,38 @@ pub struct JapaneseDictionary {
     _lexicon: LexiconSet<'static>,
 }
 
-fn map_file(path: &Path) -> SudachiResult<FileMapping> {
+fn map_file(path: &Path) -> SudachiResult<Storage> {
     let file = File::open(path)?;
     let mapping = unsafe { Mmap::map(&file) }?;
-    Ok(FileMapping { file, mapping })
+    Ok(Storage::File(mapping))
 }
 
-fn load_system_dic(cfg: &Config) -> SudachiResult<FileMapping> {
+fn load_system_dic(cfg: &Config) -> SudachiResult<Storage> {
     match &cfg.system_dict {
         Some(p) => map_file(p),
         None => return Err(ConfigError(MissingArgument(String::from("system_dict")))),
     }
 }
-
-fn load_user_dics(cfg: &Config) -> SudachiResult<Vec<FileMapping>> {
-    cfg.user_dicts.iter().map(|p| map_file(p)).collect()
-}
-
 impl JapaneseDictionary {
+    /// Creates a dictionary from the specified configuration
+    /// Dictionaries will be read from disk
     pub fn from_cfg(cfg: &Config) -> SudachiResult<JapaneseDictionary> {
-        let sys_dic = load_system_dic(cfg)?;
-        let user_dics = load_user_dics(cfg)?;
+        let mut sb = SudachiDicData::new(load_system_dic(cfg)?);
+
+        for udic in cfg.user_dicts.iter() {
+            sb.add_user(map_file(udic.as_path())?)
+        }
+
+        Self::from_cfg_storage(cfg, sb)
+    }
 
+    /// Creats a dictionary from the specified configuration and storage
+    pub fn from_cfg_storage(
+        cfg: &Config,
+        storage: SudachiDicData,
+    ) -> SudachiResult<JapaneseDictionary> {
         let mut basic_dict = LoadedDictionary::from_system_dictionary(
-            unsafe { sys_dic.as_static_slice() },
+            unsafe { storage.system_static_slice() },
             &cfg.character_definition_file,
         )?;
 
@@ -138,17 +97,14 @@ impl JapaneseDictionary {
         }
 
         let mut dic = JapaneseDictionary {
-            backend: StorageBackend::FileSystem {
-                system: sys_dic,
-                user: user_dics,
-            },
-            plugins: plugins,
+            storage,
+            plugins,
             _grammar: basic_dict.grammar,
             _lexicon: basic_dict.lexicon_set,
         };
 
         // this Vec is needed to prevent double borrowing of dic
-        let user_dicts: Vec<_> = dic.backend.user_dicts_iter().collect();
+        let user_dicts: Vec<_> = dic.storage.user_static_slice();
         for udic in user_dicts {
             dic = dic.merge_user_dictionary(udic)?;
         }
diff --git a/sudachi/src/dic/mod.rs b/sudachi/src/dic/mod.rs
index 5698909..323f314 100644
--- a/sudachi/src/dic/mod.rs
+++ b/sudachi/src/dic/mod.rs
@@ -38,6 +38,7 @@ pub mod header;
 pub mod lexicon;
 pub mod lexicon_set;
 pub mod read;
+pub mod storage;
 pub mod subset;
 pub mod word_id;
 
diff --git a/sudachi/src/dic/storage.rs b/sudachi/src/dic/storage.rs
new file mode 100644
index 0000000..b724ccb
--- /dev/null
+++ b/sudachi/src/dic/storage.rs
@@ -0,0 +1,71 @@
+/*
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+use memmap2::Mmap;
+use nom::AsBytes;
+
+pub enum Storage {
+    File(Mmap),
+    Borrowed(&'static [u8]),
+    Owned(Vec<u8>),
+}
+
+impl AsRef<[u8]> for Storage {
+    fn as_ref(&self) -> &[u8] {
+        match self {
+            Storage::File(m) => m.as_bytes(),
+            Storage::Borrowed(b) => b,
+            Storage::Owned(v) => &v,
+        }
+    }
+}
+
+pub struct SudachiDicData {
+    // system dictionary
+    system: Storage,
+    // user dictionaries
+    user: Vec<Storage>,
+}
+
+impl SudachiDicData {
+    pub fn new(system: Storage) -> Self {
+        Self {
+            system,
+            user: Vec::new(),
+        }
+    }
+
+    pub fn add_user(&mut self, user: Storage) {
+        self.user.push(user)
+    }
+
+    pub fn system(&self) -> &[u8] {
+        self.system.as_ref()
+    }
+
+    pub unsafe fn system_static_slice(&self) -> &'static [u8] {
+        std::mem::transmute(self.system())
+    }
+
+    pub(crate) fn user_static_slice(&self) -> Vec<&'static [u8]> {
+        let mut result = Vec::with_capacity(self.user.len());
+        for u in self.user.iter() {
+            let slice: &'static [u8] = unsafe { std::mem::transmute(u.as_ref()) };
+            result.push(slice);
+        }
+        result
+    }
+}
diff --git a/sudachi/src/input_text/buffer/edit.rs b/sudachi/src/input_text/buffer/edit.rs
index c1967a7..300f73a 100644
--- a/sudachi/src/input_text/buffer/edit.rs
+++ b/sudachi/src/input_text/buffer/edit.rs
@@ -17,11 +17,13 @@
 use crate::input_text::buffer::REALLY_MAX_LENGTH;
 use std::ops::Range;
 
+#[derive(Clone)]
 pub struct ReplaceOp<'a> {
     what: Range<usize>,
     with: ReplaceTgt<'a>,
 }
 
+#[derive(Clone)]
 enum ReplaceTgt<'a> {
     Ref(&'a str),
     Char(char),
diff --git a/sudachi/src/input_text/buffer/mod.rs b/sudachi/src/input_text/buffer/mod.rs
index dce5006..91dfa28 100644
--- a/sudachi/src/input_text/buffer/mod.rs
+++ b/sudachi/src/input_text/buffer/mod.rs
@@ -34,7 +34,7 @@ const MAX_LENGTH: usize = u16::MAX as usize / 4 * 3;
 /// if the limit of the rewritten sentence is more than this number, then all bets are off
 const REALLY_MAX_LENGTH: usize = u16::MAX as usize;
 
-#[derive(Eq, PartialEq, Debug)]
+#[derive(Eq, PartialEq, Debug, Clone)]
 enum BufferState {
     Clean,
     RW,
@@ -51,7 +51,7 @@ impl Default for BufferState {
 ///
 /// By saying char we actually mean Unicode codepoint here.
 /// In the context of this struct these terms are synonyms.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct InputBuffer {
     /// Original input data, output is done on this
     original: String,


Test patch:
diff --git a/python/tests/test_morpheme.py b/python/tests/test_morpheme.py
index b9600bc..063fd6e 100644
--- a/python/tests/test_morpheme.py
+++ b/python/tests/test_morpheme.py
@@ -115,6 +115,23 @@ class TestTokenizer(unittest.TestCase):
         m = self.tokenizer_obj.tokenize('東京府')[0]
         self.assertEqual(m.synonym_group_ids(), [1, 3])
 
+    def test_normalize_half_full(self):
+        m = self.tokenizer_obj.tokenize('特Ａ東京')
+        self.assertEqual(len(m), 2)
+
+        self.assertEqual(m[0].surface(), "特Ａ")
+        self.assertEqual(m[0].begin(), 0)
+        self.assertEqual(m[0].end(), 2)
+
+    def test_normalize_katakana_half_full(self):
+        m = self.tokenizer_obj.tokenize('ｶﾞ5')
+        self.assertEqual(len(m), 2)
+
+        self.assertEqual(m[0].surface(), "ｶﾞ")
+        self.assertEqual(m[0].end(), 2)
+        self.assertEqual(m[1].begin(), 2)
+        self.assertEqual(len(m[0]), 2)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/sudachi/tests/common/mod.rs b/sudachi/tests/common/mod.rs
index a771278..8c13bc0 100644
--- a/sudachi/tests/common/mod.rs
+++ b/sudachi/tests/common/mod.rs
@@ -14,21 +14,26 @@
  * limitations under the License.
  */
 
-use std::path::{Path, PathBuf};
-
 extern crate sudachi;
-use self::sudachi::analysis::stateful_tokenizer::StatefulTokenizer;
-use self::sudachi::dic::dictionary::JapaneseDictionary;
+
 use std::fs;
 use std::fs::File;
 use std::io::Read;
+use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::sync::Arc;
+
+use sudachi::analysis::stateful_tokenizer::StatefulTokenizer;
 use sudachi::analysis::stateless_tokenizer::StatelessTokenizer;
 use sudachi::config::Config;
-use sudachi::dic::{grammar::Grammar, header::Header, lexicon::Lexicon};
+use sudachi::dic::dictionary::JapaneseDictionary;
+use sudachi::dic::{grammar::Grammar, header::Header, lexicon::Lexicon, DictionaryLoader};
 use sudachi::prelude::*;
 
+use lazy_static::lazy_static;
+use sudachi::dic::build::DictBuilder;
+use sudachi::dic::storage::{Storage, SudachiDicData};
+
 pub fn dictionary_bytes_from_path<P: AsRef<Path>>(dictionary_path: P) -> SudachiResult<Vec<u8>> {
     let dictionary_path = dictionary_path.as_ref();
     let dictionary_stat = fs::metadata(&dictionary_path)?;
@@ -105,14 +110,80 @@ impl TestTokenizer {
     }
 }
 
+pub struct TestTokenizerBuilder<'a> {
+    pub conn: Option<&'a [u8]>,
+    pub system: &'a [u8],
+    pub user: Vec<&'a [u8]>,
+    pub mode: Mode,
+    pub debug: bool,
+}
+
+#[allow(unused)]
+impl<'a> TestTokenizerBuilder<'a> {
+    pub fn user(mut self, data: &'a [u8]) -> Self {
+        self.user.push(data);
+        self
+    }
+
+    pub fn mode(mut self, mode: Mode) -> Self {
+        self.mode = mode;
+        self
+    }
+
+    pub fn debug(mut self, debug: bool) -> Self {
+        self.debug = debug;
+        self
+    }
+
+    pub fn build(self) -> TestStatefulTokenizer {
+        let mut sys = DictBuilder::new_system();
+        sys.read_conn(
+            self.conn
+                .unwrap_or(include_bytes!("../resources/matrix_10x10.def")),
+        )
+        .unwrap();
+        sys.read_lexicon(self.system).unwrap();
+        sys.resolve().unwrap();
+        let mut sys_bytes = Vec::new();
+        sys.compile(&mut sys_bytes).unwrap();
+
+        let mut data = SudachiDicData::new(Storage::Owned(sys_bytes));
+
+        if !self.user.is_empty() {
+            let dic =
+                DictionaryLoader::read_system_dictionary(unsafe { data.system_static_slice() })
+                    .unwrap()
+                    .to_loaded()
+                    .unwrap();
+
+            for u in self.user {
+                let mut ubld = DictBuilder::new_user(&dic);
+                ubld.read_lexicon(u).unwrap();
+                ubld.resolve().unwrap();
+                let mut user_bytes = Vec::new();
+                ubld.compile(&mut user_bytes).unwrap();
+                data.add_user(Storage::Owned(user_bytes));
+            }
+        }
+
+        let dic = JapaneseDictionary::from_cfg_storage(&TEST_CONFIG, data).unwrap();
+        let rcdic = Rc::new(dic);
+
+        TestStatefulTokenizer {
+            tok: StatefulTokenizer::create(rcdic.clone(), self.debug, self.mode),
+            result: MorphemeList::empty(rcdic),
+        }
+    }
+}
+
 pub struct TestStatefulTokenizer {
-    tok: StatefulTokenizer<Rc<JapaneseDictionary>>,
-    result: MorphemeList<Rc<JapaneseDictionary>>,
+    pub tok: StatefulTokenizer<Rc<JapaneseDictionary>>,
+    pub result: MorphemeList<Rc<JapaneseDictionary>>,
 }
 
 #[allow(unused)]
 impl TestStatefulTokenizer {
-    pub fn new(mode: Mode) -> TestStatefulTokenizer {
+    pub fn new_built(mode: Mode) -> TestStatefulTokenizer {
         let dic = Rc::new(JapaneseDictionary::from_cfg(&TEST_CONFIG).expect("works"));
         Self {
             tok: StatefulTokenizer::new(dic.clone(), mode),
@@ -120,6 +191,16 @@ impl TestStatefulTokenizer {
         }
     }
 
+    pub fn builder(system: &[u8]) -> TestTokenizerBuilder {
+        TestTokenizerBuilder {
+            system,
+            user: Vec::new(),
+            conn: None,
+            mode: Mode::C,
+            debug: false,
+        }
+    }
+
     pub fn tokenize(&mut self, data: &str) -> &MorphemeList<Rc<JapaneseDictionary>> {
         self.tok.reset().push_str(data);
         self.tok.do_tokenize().expect("tokenization failed");
diff --git a/sudachi/tests/dictionary.rs b/sudachi/tests/dictionary.rs
index 5c1df0c..241eb8f 100644
--- a/sudachi/tests/dictionary.rs
+++ b/sudachi/tests/dictionary.rs
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-#[macro_use]
 extern crate lazy_static;
 
 mod common;
diff --git a/sudachi/tests/dictionary_header.rs b/sudachi/tests/dictionary_header.rs
index 8e4843a..0cb63f5 100644
--- a/sudachi/tests/dictionary_header.rs
+++ b/sudachi/tests/dictionary_header.rs
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-#[macro_use]
 extern crate lazy_static;
 
 extern crate sudachi;
diff --git a/sudachi/tests/lexicon.rs b/sudachi/tests/lexicon.rs
index cd2d014..d8d9f91 100644
--- a/sudachi/tests/lexicon.rs
+++ b/sudachi/tests/lexicon.rs
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-#[macro_use]
 extern crate lazy_static;
 
 mod common;
diff --git a/sudachi/tests/morpheme.rs b/sudachi/tests/morpheme.rs
index 98b0436..3b1e3dd 100644
--- a/sudachi/tests/morpheme.rs
+++ b/sudachi/tests/morpheme.rs
@@ -14,7 +14,6 @@
  *  limitations under the License.
  */
 
-#[macro_use]
 extern crate lazy_static;
 extern crate sudachi;
 
diff --git a/sudachi/tests/resources/matrix_10x10.def b/sudachi/tests/resources/matrix_10x10.def
new file mode 100644
index 0000000..de3529a
--- /dev/null
+++ b/sudachi/tests/resources/matrix_10x10.def
@@ -0,0 +1,101 @@
+10 10
+0 0 0
+0 1 863
+0 2 2124
+0 3 1032
+0 4 591
+0 5 -162
+0 6 -79
+0 7 887
+0 8 447
+0 9 -535
+1 0 -3689
+1 1 -3361
+1 2 -7643
+1 3 -3267
+1 4 809
+1 5 -1098
+1 6 4606
+1 7 4269
+1 8 4567
+1 9 1635
+2 0 -1959
+2 1 2457
+2 2 811
+2 3 840
+2 4 903
+2 5 -958
+2 6 517
+2 7 2037
+2 8 1392
+2 9 -193
+3 0 -2288
+3 1 1741
+3 2 487
+3 3 792
+3 4 -1474
+3 5 -3429
+3 6 126
+3 7 437
+3 8 605
+3 9 -547
+4 0 -2809
+4 1 -3584
+4 2 -6743
+4 3 -2869
+4 4 -2805
+4 5 -407
+4 6 3422
+4 7 5642
+4 8 6382
+4 9 2165
+5 0 -509
+5 1 -3665
+5 2 -3882
+5 3 -572
+5 4 -1036
+5 5 -54
+5 6 2570
+5 7 3319
+5 8 4059
+5 9 882
+6 0 101
+6 1 2933
+6 2 2198
+6 3 -2004
+6 4 4392
+6 5 4017
+6 6 569
+6 7 475
+6 8 -390
+6 9 852
+7 0 -852
+7 1 2079
+7 2 1180
+7 3 -3084
+7 4 2010
+7 5 1570
+7 6 746
+7 7 2341
+7 8 2051
+7 9 1393
+8 0 -522
+8 1 3354
+8 2 2037
+8 3 -2542
+8 4 3071
+8 5 2631
+8 6 -352
+8 7 2847
+8 8 1134
+8 9 1256
+9 0 -975
+9 1 2498
+9 2 1690
+9 3 -1523
+9 4 3023
+9 5 3139
+9 6 2562
+9 7 3962
+9 8 418
+9 9 -2490
diff --git a/sudachi/tests/resources/split_alpha.csv b/sudachi/tests/resources/split_alpha.csv
new file mode 100644
index 0000000..af72735
--- /dev/null
+++ b/sudachi/tests/resources/split_alpha.csv
@@ -0,0 +1,3 @@
+ab,6,6,1000,AB,名詞,普通名詞,一般,*,*,*,AB,AB,*,C,1/2,1/2,*,*
+a,-1,6,1000,A,名詞,数詞,*,*,*,*,A,A,*,A,*,*,*,*
+b,-1,6,1000,B,名詞,固有名詞,地名,一般,*,*,B,B,*,A,*,*,*,*
\ No newline at end of file
diff --git a/sudachi/tests/sentence_detector.rs b/sudachi/tests/sentence_detector.rs
index a9c5b6c..cbf79df 100644
--- a/sudachi/tests/sentence_detector.rs
+++ b/sudachi/tests/sentence_detector.rs
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-#[macro_use]
 extern crate lazy_static;
 
 mod common;
diff --git a/sudachi/tests/split.rs b/sudachi/tests/split.rs
new file mode 100644
index 0000000..426552a
--- /dev/null
+++ b/sudachi/tests/split.rs
@@ -0,0 +1,41 @@
+/*
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+use crate::common::TestStatefulTokenizer;
+use sudachi::prelude::Mode;
+
+mod common;
+
+const AB_DIC: &[u8] = include_bytes!("resources/split_alpha.csv");
+
+#[test]
+fn split_at_analysis_time() {
+    let mut tok = TestStatefulTokenizer::builder(AB_DIC).mode(Mode::A).build();
+    let res = tok.tokenize("ＡＢ");
+    assert_eq!(res.len(), 2);
+    assert_eq!(res.get(0).surface(), "Ａ");
+    assert_eq!(res.get(1).surface(), "Ｂ");
+}
+
+#[test]
+fn split_after_analysis() {
+    let mut tok = TestStatefulTokenizer::builder(AB_DIC).build();
+    let res = tok.tokenize("ＡＢ");
+    assert_eq!(res.len(), 1);
+    let res2 = res.get(0).split(Mode::A).unwrap();
+    assert_eq!(res2.get(0).surface(), "Ａ");
+    assert_eq!(res2.get(1).surface(), "Ｂ");
+}
diff --git a/sudachi/tests/stateful_tokenizer.rs b/sudachi/tests/stateful_tokenizer.rs
index 36d1c48..ff00735 100644
--- a/sudachi/tests/stateful_tokenizer.rs
+++ b/sudachi/tests/stateful_tokenizer.rs
@@ -14,7 +14,6 @@
  *  limitations under the License.
  */
 
-#[macro_use]
 extern crate lazy_static;
 extern crate sudachi;
 
@@ -26,21 +25,21 @@ use crate::common::TestStatefulTokenizer as TestTokenizer;
 
 #[test]
 fn empty() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("");
     assert_eq!(0, ms.len());
 }
 
 #[test]
 fn tokenize_small_katakana_only() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("ァ");
     assert_eq!(1, ms.len());
 }
 
 #[test]
 fn get_word_id() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("京都");
     assert_eq!(1, ms.len());
     let m0 = ms.get(0);
@@ -57,7 +56,7 @@ fn get_word_id() {
 
 #[test]
 fn get_dictionary_id() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("京都");
     let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
@@ -76,7 +75,7 @@ fn get_dictionary_id() {
 
 #[test]
 fn get_synonym_group_id() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("京都");
     assert_eq!(1, ms.len());
     assert_eq!([1, 5], ms.get(0).synonym_group_ids());
@@ -92,7 +91,7 @@ fn get_synonym_group_id() {
 
 #[test]
 fn tokenize_kanji_alphabet_word() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     assert_eq!(1, tok.tokenize("特a").len());
     assert_eq!(1, tok.tokenize("ab").len());
     assert_eq!(2, tok.tokenize("特ab").len());
@@ -100,7 +99,7 @@ fn tokenize_kanji_alphabet_word() {
 
 #[test]
 fn tokenize_with_dots() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("京都…");
     assert_eq!(4, ms.len());
     assert_eq!("…", ms.get(1).surface());
@@ -113,7 +112,7 @@ fn tokenize_with_dots() {
 
 #[test]
 fn tokenizer_morpheme_split() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("東京都");
     assert_eq!(1, ms.len());
     assert_eq!("東京都", ms.get(0).surface());
@@ -127,7 +126,7 @@ fn tokenizer_morpheme_split() {
 
 #[test]
 fn split_middle() {
-    let mut tok = TestTokenizer::new(Mode::C);
+    let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("京都東京都京都");
     assert_eq!(ms.len(), 3);
     let m = ms.get(1);
diff --git a/sudachi/tests/stateless_tokenizer.rs b/sudachi/tests/stateless_tokenizer.rs
index 59ad739..4b1d904 100644
--- a/sudachi/tests/stateless_tokenizer.rs
+++ b/sudachi/tests/stateless_tokenizer.rs
@@ -14,7 +14,6 @@
  *  limitations under the License.
  */
 
-#[macro_use]
 extern crate lazy_static;
 extern crate sudachi;