Repository: WorksApplications/sudachi.rs (language: python)

Issue:
Pyhon API: ability to reduce allocations by reusing objects
`Tokenizer.analyze` and `Morpheme.split` will have new out parameter which will accept `MorphemeList`. It will be rewritten with new content. Passing the same MorphemeList twice will be diagnosed and an exception will be thrown.


Discussion / hints:

Fix patch:
diff --git a/python/py_src/sudachipy/sudachipy.pyi b/python/py_src/sudachipy/sudachipy.pyi
index cab09b23..4ee32dc7 100644
--- a/python/py_src/sudachipy/sudachipy.pyi
+++ b/python/py_src/sudachipy/sudachipy.pyi
@@ -153,7 +153,7 @@ class Morpheme:
         """
         ...
 
-    def split(self, mode: SplitMode) -> MorphemeList:
+    def split(self, mode: SplitMode, out: Optional[MorphemeList] = None) -> MorphemeList:
         """
         Returns a list of morphemes splitting itself with given split mode.
         """
@@ -229,15 +229,17 @@ class SplitMode:
     B: ClassVar[SplitMode] = ...
     C: ClassVar[SplitMode] = ...
     @classmethod
-    def __init__(self) -> None: ...
+    def __init__(cls) -> None: ...
 
 
 class Tokenizer:
     SplitMode: ClassVar[sudachipy.SplitMode] = ...
     @classmethod
-    def __init__(self) -> None: ...
+    def __init__(cls) -> None: ...
 
-    def tokenize(self, text: str, mode: sudachipy.SplitMode = ...) -> MorphemeList:
+    def tokenize(self, text: str,
+                 mode: sudachipy.SplitMode = ...,
+                 out: Optional[MorphemeList] = None) -> MorphemeList:
         """
         Break text into morphemes.
 
diff --git a/python/src/morpheme.rs b/python/src/morpheme.rs
index dfb01413..237b6bf0 100644
--- a/python/src/morpheme.rs
+++ b/python/src/morpheme.rs
@@ -14,14 +14,14 @@
  *  limitations under the License.
  */
 
+use std::ops::Deref;
 use std::sync::Arc;
 
 use pyo3::exceptions::{self, PyException};
 use pyo3::prelude::*;
-use pyo3::types::{PyList, PyTuple, PyType};
+use pyo3::types::{PyList, PyString, PyTuple, PyType};
 
-use sudachi::analysis::morpheme::MorphemeList;
-use sudachi::analysis::node::LatticeNode;
+use sudachi::prelude::{Morpheme, MorphemeList};
 
 use crate::dictionary::{PyDicData, PyDictionary};
 use crate::tokenizer::PySplitMode;
@@ -33,9 +33,39 @@ type PyMorphemeList = MorphemeList<Arc<PyDicData>>;
 #[pyclass(module = "sudachipy.morphemelist", name = "MorphemeList")]
 #[repr(transparent)]
 pub struct PyMorphemeListWrapper {
-    inner: Arc<PyMorphemeList>,
+    /// use `internal()` function instead
+    inner: PyMorphemeList,
 }
 
+// PyMorphemeListWrapper is used only when GIL is active,
+// all associated functions take GIL token as a parameter
+unsafe impl Sync for PyMorphemeListWrapper {}
+unsafe impl Send for PyMorphemeListWrapper {}
+
+impl PyMorphemeListWrapper {
+    pub(crate) fn new(dict: Arc<PyDicData>) -> Self {
+        Self {
+            inner: PyMorphemeList::empty(dict),
+        }
+    }
+
+    /// Borrow internals mutable. GIL token proves access.
+    pub(crate) fn internal_mut(&mut self, _py: Python) -> &mut PyMorphemeList {
+        &mut self.inner
+    }
+
+    /// Borrow internals immutable. GIL token proves access.
+    pub(crate) fn internal(&self, _py: Python) -> &PyMorphemeList {
+        &self.inner
+    }
+
+    /// Create a copy with empty list of Nodes. GIL token proves access.
+    pub(crate) fn empty_clone(&self, _py: Python) -> Self {
+        Self {
+            inner: self.inner.empty_clone(),
+        }
+    }
+}
 #[pymethods]
 impl PyMorphemeListWrapper {
     /// Returns an empty morpheme list with dictionary
@@ -46,207 +76,259 @@ impl PyMorphemeListWrapper {
         PyErr::warn(py, cat, "Users should not generate MorphemeList by themselves. Use Tokenizer.tokenize(\"\") if you need.", 1)?;
 
         Ok(Self {
-            inner: Arc::new(PyMorphemeList::empty(
-                dict.dictionary.as_ref().unwrap().clone(),
-            )),
+            inner: PyMorphemeList::empty(dict.dictionary.as_ref().unwrap().clone()),
         })
     }
 
     /// Returns the total cost of the path
     #[pyo3(text_signature = "($self)")]
-    fn get_internal_cost(&self) -> i32 {
-        self.inner.get_internal_cost()
+    fn get_internal_cost(&self, py: Python) -> i32 {
+        self.internal(py).get_internal_cost()
     }
 
     /// Returns the number of morpheme in this list.
     #[pyo3(text_signature = "($self)")]
-    fn size(&self) -> usize {
-        self.inner.len()
+    fn size(&self, py: Python) -> usize {
+        self.internal(py).len()
     }
-}
 
-impl From<MorphemeList<Arc<PyDicData>>> for PyMorphemeListWrapper {
-    fn from(morpheme_list: MorphemeList<Arc<PyDicData>>) -> Self {
-        Self {
-            inner: Arc::new(morpheme_list),
-        }
+    fn __len__(&self, py: Python) -> usize {
+        self.size(py)
     }
-}
 
-#[pyproto]
-impl pyo3::basic::PyObjectProtocol for PyMorphemeListWrapper {
-    fn __str__(&self) -> &str {
-        self.inner.surface()
-    }
-}
+    fn __getitem__(slf: &PyCell<PyMorphemeListWrapper>, mut idx: isize) -> PyResult<PyMorpheme> {
+        let list = slf.borrow();
+        let py = slf.py();
+        let len = list.size(py) as isize;
 
-#[pyproto]
-impl pyo3::sequence::PySequenceProtocol for PyMorphemeListWrapper {
-    fn __len__(&self) -> usize {
-        self.size()
-    }
+        if idx < 0 {
+            // negative indexing
+            idx += len;
+        }
 
-    fn __getitem__(&self, idx: isize) -> PyResult<PyMorpheme> {
-        // pyo3 automatically adds len when a negative idx is given
-        let len = self.__len__() as isize;
         if idx < 0 || len <= idx {
             return Err(PyErr::new::<exceptions::PyIndexError, _>(format!(
                 "morphemelist index out of range: the len is {} but the index is {}",
-                self.__len__(),
+                list.size(py),
                 idx
             )));
         }
 
+        let py_list: Py<PyMorphemeListWrapper> = slf.into();
+
         Ok(PyMorpheme {
-            list: self.inner.clone(),
+            list: py_list,
             index: idx as usize,
         })
     }
+
+    fn __str__<'py>(&'py self, py: Python<'py>) -> &PyString {
+        PyString::new(py, self.internal(py).surface().deref())
+    }
+
+    fn __iter__(slf: Py<Self>) -> PyMorphemeIter {
+        PyMorphemeIter {
+            list: slf,
+            index: 0,
+        }
+    }
+
+    fn __bool__(&self, py: Python) -> bool {
+        self.internal(py).len() != 0
+    }
 }
 
-#[pyproto]
-impl pyo3::iter::PyIterProtocol for PyMorphemeListWrapper {
-    fn __iter__(slf: PyRef<Self>) -> PyResult<Py<PyMorphemeIter>> {
-        Py::new(
-            slf.py(),
-            PyMorphemeIter {
-                list: slf.inner.clone(),
-                index: 0,
-            },
-        )
+impl From<MorphemeList<Arc<PyDicData>>> for PyMorphemeListWrapper {
+    fn from(morpheme_list: MorphemeList<Arc<PyDicData>>) -> Self {
+        Self {
+            inner: morpheme_list,
+        }
     }
 }
 
 /// A morpheme (basic semantic unit of language).
 #[pyclass(module = "sudachipy.morphemelist", name = "MorphemeIter")]
 pub struct PyMorphemeIter {
-    list: Arc<PyMorphemeList>,
+    list: Py<PyMorphemeListWrapper>,
     index: usize,
 }
 
-#[pyproto]
-impl pyo3::iter::PyIterProtocol for PyMorphemeIter {
+#[pymethods]
+impl PyMorphemeIter {
     fn __iter__(slf: PyRef<Self>) -> PyRef<Self> {
         slf
     }
 
-    fn __next__(mut slf: PyRefMut<Self>) -> Option<PyMorpheme> {
-        if slf.index >= slf.list.len() {
+    fn __next__(&mut self, py: Python) -> Option<PyMorpheme> {
+        if self.index >= self.list.borrow(py).size(py) {
             return None;
         }
 
         let morpheme = PyMorpheme {
-            list: slf.list.clone(),
-            index: slf.index,
+            list: self.list.clone(),
+            index: self.index,
         };
 
-        slf.index += 1;
+        self.index += 1;
         Some(morpheme)
     }
 }
 
+/// It is a syntax sugar for accessing Morpheme reference
+/// Without it binding implementations become much less readable
+struct MorphemeRef<'py> {
+    #[allow(unused)] // need to keep this around for correct reference count
+    list: PyRef<'py, PyMorphemeListWrapper>,
+    morph: Morpheme<'py, Arc<PyDicData>>,
+}
+
+impl<'py> Deref for MorphemeRef<'py> {
+    type Target = Morpheme<'py, Arc<PyDicData>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.morph
+    }
+}
+
 #[pyclass(module = "sudachipy.morpheme", name = "Morpheme")]
 pub struct PyMorpheme {
-    list: Arc<PyMorphemeList>,
+    list: Py<PyMorphemeListWrapper>,
     index: usize,
 }
 
-#[pyproto]
-impl pyo3::basic::PyObjectProtocol for PyMorpheme {
-    fn __str__(&self) -> &str {
-        self.surface()
+impl PyMorpheme {
+    fn list<'py>(&'py self, py: Python<'py>) -> PyRef<'py, PyMorphemeListWrapper> {
+        self.list.borrow(py)
+    }
+
+    fn morph<'py>(&'py self, py: Python<'py>) -> MorphemeRef<'py> {
+        let list = self.list(py);
+        // workaround for self-referential structs
+        let morph = unsafe { std::mem::transmute(list.internal(py).get(self.index)) };
+        MorphemeRef { list, morph }
     }
 }
 
 #[pymethods]
 impl PyMorpheme {
+    fn __str__<'py>(&'py self, py: Python<'py>) -> PyObject {
+        self.surface(py)
+    }
+
     /// Returns the begin index of this in the input text
     #[pyo3(text_signature = "($self)")]
-    fn begin(&self) -> usize {
+    fn begin(&self, py: Python) -> usize {
         // call codepoint version
-        self.list.get_begin_cp(self.index)
+        self.morph(py).begin_c()
     }
 
     /// Returns the end index of this in the input text
     #[pyo3(text_signature = "($self)")]
-    fn end(&self) -> usize {
+    fn end(&self, py: Python) -> usize {
         // call codepoint version
-        self.list.get_end_cp(self.index)
+        self.morph(py).end_c()
     }
 
     /// Returns the surface
     #[pyo3(text_signature = "($self)")]
-    fn surface(&self) -> &str {
-        self.list.get_surface(self.index)
+    fn surface<'py>(&'py self, py: Python<'py>) -> PyObject {
+        self.morph(py).surface().deref().into_py(py)
     }
 
     /// Returns the part of speech
     #[pyo3(text_signature = "($self)")]
-    fn part_of_speech<'py>(&'py self, py: Python<'py>) -> &'py PyTuple {
-        let pos_id = self.part_of_speech_id();
-        self.list.dic().pos_of(pos_id).as_ref(py)
+    fn part_of_speech<'py>(&'py self, py: Python<'py>) -> Py<PyTuple> {
+        let pos_id = self.part_of_speech_id(py);
+        self.list(py)
+            .internal(py)
+            .dict()
+            .pos_of(pos_id)
+            .clone_ref(py)
     }
 
     /// Returns the id of the part of speech in the dictionary
     #[pyo3(text_signature = "($self)")]
-    pub fn part_of_speech_id(&self) -> u16 {
-        self.list.get_word_info(self.index).pos_id()
+    pub fn part_of_speech_id(&self, py: Python) -> u16 {
+        self.morph(py).part_of_speech_id()
     }
 
     /// Returns the dictionary form
     #[pyo3(text_signature = "($self)")]
-    fn dictionary_form(&self) -> &str {
-        self.list.get_word_info(self.index).dictionary_form()
+    fn dictionary_form<'py>(&'py self, py: Python<'py>) -> PyObject {
+        self.morph(py).get_word_info().dictionary_form().into_py(py)
     }
 
     /// Returns the normalized form
     #[pyo3(text_signature = "($self)")]
-    fn normalized_form(&self) -> &str {
-        self.list.get_word_info(self.index).normalized_form()
+    fn normalized_form<'py>(&'py self, py: Python<'py>) -> PyObject {
+        self.morph(py).get_word_info().normalized_form().into_py(py)
     }
 
     /// Returns the reading form
     #[pyo3(text_signature = "($self)")]
-    fn reading_form(&self) -> &str {
-        self.list.get_word_info(self.index).reading_form()
+    fn reading_form<'py>(&'py self, py: Python<'py>) -> PyObject {
+        self.morph(py).get_word_info().reading_form().into_py(py)
     }
 
     /// Returns a list of morphemes splitting itself with given split mode
-    #[pyo3(text_signature = "($self, mode: sudachipy.SplitMode) -> sudachipy.MorphemeList")]
-    fn split(&self, py: Python, mode: PySplitMode) -> PyResult<PyMorphemeListWrapper> {
-        let cat = PyModule::import(py, "builtins")?.getattr("DeprecationWarning")?;
-        PyErr::warn(
-            py,
-            cat,
-            "API around this functionality will change. See github issue #92 for more.",
-            1,
-        )?;
-
-        Ok(self
-            .list
-            .split(mode.into(), self.index)
+    #[pyo3(
+        text_signature = "($self, mode: sudachipy.SplitMode, out = None) -> sudachipy.MorphemeList"
+    )]
+    fn split<'py>(
+        &'py self,
+        py: Python<'py>,
+        mode: PySplitMode,
+        out: Option<&'py PyCell<PyMorphemeListWrapper>>,
+        add_single: Option<bool>,
+    ) -> PyResult<&'py PyCell<PyMorphemeListWrapper>> {
+        let list = self.list(py);
+
+        let out_cell = match out {
+            None => {
+                let list = list.empty_clone(py);
+                PyCell::new(py, list)?
+            }
+            Some(r) => r,
+        };
+
+        let mut borrow = out_cell.try_borrow_mut();
+        let out_ref = match borrow {
+            Ok(ref mut v) => v.internal_mut(py),
+            Err(_) => return Err(PyException::new_err("out was used twice")),
+        };
+
+        out_ref.clear();
+        let splitted = list
+            .internal(py)
+            .split_into(mode.into(), self.index, out_ref)
             .map_err(|e| {
                 PyException::new_err(format!("Error while splitting morpheme: {}", e.to_string()))
-            })?
-            .into())
+            })?;
+
+        if add_single.unwrap_or(true) && !splitted {
+            list.internal(py)
+                .copy_slice(self.index, self.index + 1, out_ref);
+        }
+
+        Ok(out_cell)
     }
 
     /// Returns whether if this is out of vocabulary word
     #[pyo3(text_signature = "($self)")]
-    fn is_oov(&self) -> bool {
-        self.list.is_oov(self.index)
+    fn is_oov(&self, py: Python) -> bool {
+        self.morph(py).is_oov()
     }
 
     /// Returns word id of this word in the dictionary
     #[pyo3(text_signature = "($self)")]
-    fn word_id(&self) -> u32 {
-        self.list.get_node(self.index).word_id().as_raw()
+    fn word_id(&self, py: Python) -> u32 {
+        self.morph(py).word_id().as_raw()
     }
 
     /// Returns the dictionary id which this word belongs
     #[pyo3(text_signature = "($self)")]
-    fn dictionary_id(&self) -> i32 {
-        let word_id = self.list.get_node(self.index).word_id();
+    fn dictionary_id(&self, py: Python) -> i32 {
+        let word_id = self.morph(py).word_id();
         if word_id.is_oov() {
             -1
         } else {
@@ -256,9 +338,10 @@ impl PyMorpheme {
 
     /// Returns the list of synonym group ids
     #[pyo3(text_signature = "($self)")]
-    fn synonym_group_ids(&self, py: Python) -> Py<PyList> {
-        let ids = self.list.get_word_info(self.index).synonym_group_ids();
-        PyList::new(py, ids).into()
+    fn synonym_group_ids<'py>(&'py self, py: Python<'py>) -> &'py PyList {
+        let mref = self.morph(py);
+        let ids = mref.get_word_info().synonym_group_ids();
+        PyList::new(py, ids)
     }
 
     /// Returns the word info
@@ -267,11 +350,12 @@ impl PyMorpheme {
         let cat = PyModule::import(py, "builtins")?.getattr("DeprecationWarning")?;
         PyErr::warn(py, cat, "Users should not touch the raw WordInfo.", 1)?;
 
-        Ok(self.list.get_word_info(self.index).clone().into())
+        Ok(self.morph(py).get_word_info().clone().into())
     }
 
     /// Returns morpheme length in codepoints
-    pub fn __len__(&self) -> usize {
-        self.end() - self.begin()
+    pub fn __len__(&self, py: Python) -> usize {
+        let m = self.morph(py);
+        m.end_c() - m.begin_c()
     }
 }
diff --git a/python/src/pos_matcher.rs b/python/src/pos_matcher.rs
index 4266a9a7..26182957 100644
--- a/python/src/pos_matcher.rs
+++ b/python/src/pos_matcher.rs
@@ -115,8 +115,8 @@ impl PyPosMatcher {
 
 #[pymethods]
 impl PyPosMatcher {
-    pub fn __call__(&self, m: &PyMorpheme) -> bool {
-        let pos_id = m.part_of_speech_id();
+    pub fn __call__<'py>(&'py self, py: Python<'py>, m: &'py PyMorpheme) -> bool {
+        let pos_id = m.part_of_speech_id(py);
         self.matcher.matches_id(pos_id)
     }
 
diff --git a/python/src/pretokenizer.rs b/python/src/pretokenizer.rs
index 61be44b6..157d0fb6 100644
--- a/python/src/pretokenizer.rs
+++ b/python/src/pretokenizer.rs
@@ -21,17 +21,17 @@ use pyo3::prelude::*;
 use pyo3::types::{PyList, PySlice, PyTuple};
 use std::cell::RefCell;
 use std::sync::Arc;
-use sudachi::analysis::node::LatticeNode;
+
 use sudachi::analysis::stateful_tokenizer::StatefulTokenizer;
 use sudachi::dic::subset::InfoSubset;
-use sudachi::prelude::{Mode, MorphemeList};
+use sudachi::prelude::Mode;
 use thread_local::ThreadLocal;
 
 /// This struct perform actual tokenization
 /// There should be at most one instance per thread of execution
 struct PerThreadPreTokenizer {
     tokenizer: StatefulTokenizer<Arc<PyDicData>>,
-    morphemes: MorphemeList<Arc<PyDicData>>,
+    morphemes: Option<Py<PyMorphemeListWrapper>>,
 }
 
 impl PerThreadPreTokenizer {
@@ -40,19 +40,36 @@ impl PerThreadPreTokenizer {
         tok.set_subset(subset);
         Self {
             tokenizer: tok,
-            morphemes: MorphemeList::empty(dict.clone()),
+            morphemes: None,
         }
     }
 
     pub fn tokenize(&mut self, data: &str) -> PyResult<()> {
         self.tokenizer.reset().push_str(data);
         wrap(self.tokenizer.do_tokenize())?;
-        self.morphemes.collect_results(&mut self.tokenizer).unwrap();
         Ok(())
     }
 
-    pub fn result(&self) -> &MorphemeList<Arc<PyDicData>> {
-        &self.morphemes
+    pub fn collect_results(&mut self, py: Python) -> PyResult<()> {
+        let mut mlist = match self.morphemes.as_mut() {
+            None => {
+                self.morphemes = Some(Py::new(
+                    py,
+                    PyMorphemeListWrapper::new(self.tokenizer.dict_clone()),
+                )?);
+                self.morphemes.as_mut().unwrap().borrow_mut(py)
+            }
+            Some(ms) => ms.borrow_mut(py),
+        };
+        mlist
+            .internal_mut(py)
+            .collect_results(&mut self.tokenizer)
+            .unwrap();
+        Ok(())
+    }
+
+    pub fn result(&self) -> &Py<PyMorphemeListWrapper> {
+        self.morphemes.as_ref().unwrap()
     }
 }
 
@@ -113,14 +130,17 @@ impl PyPretokenizer {
         // tokenization itself should work without GIL, we have thread-local tokenizers here
         py.allow_threads(|| self.tokenizer_cell().borrow_mut().tokenize(input_data))?;
         // then prepare results with GIL
+        self.tokenizer_cell().borrow_mut().collect_results(py)?;
         let cell = self.tokenizer_cell().borrow();
         let morphs = cell.result();
         match self.handler.as_ref() {
             None => {
                 let result = PyList::empty(py);
+                let py_ref = morphs.borrow(py);
+                let morphs = py_ref.internal(py);
                 for idx in 0..morphs.len() {
-                    let node = morphs.get_node(idx);
-                    let slice = PySlice::new(py, node.begin() as isize, node.end() as isize, 1);
+                    let node = morphs.get(idx);
+                    let slice = PySlice::new(py, node.begin_c() as isize, node.end_c() as isize, 1);
                     let args = PyTuple::new(py, [slice]);
                     let substring = string.call_method1("slice", args)?;
                     result.append(substring)?;
@@ -128,9 +148,8 @@ impl PyPretokenizer {
                 Ok(result)
             }
             Some(h) => {
-                let mlist = PyMorphemeListWrapper::from(morphs.clone());
-                let mlist = PyCell::new(py, mlist)?;
-                let args = PyTuple::new(py, &[index, string, mlist]);
+                let mrp: &PyAny = morphs.as_ref(py);
+                let args = PyTuple::new(py, &[index, string, mrp]);
                 h.as_ref(py).call1(args)
             }
         }
diff --git a/python/src/tokenizer.rs b/python/src/tokenizer.rs
index baeff995..fdc0d502 100644
--- a/python/src/tokenizer.rs
+++ b/python/src/tokenizer.rs
@@ -95,37 +95,48 @@ impl PyTokenizer {
     /// By default tokenizer's split mode is used.
     /// The logger provided is ignored.
     #[pyo3(
-        text_signature = "($self, text: str, mode: SplitMode = None, logger = None) -> sudachipy.MorphemeList"
+        text_signature = "($self, text: str, mode: SplitMode = None, logger = None, out = None) -> sudachipy.MorphemeList"
     )]
     #[args(text, mode = "None", logger = "None")]
     #[allow(unused_variables)]
-    fn tokenize(
-        &mut self,
-        text: &str,
+    fn tokenize<'py>(
+        &'py mut self,
+        py: Python<'py>,
+        text: &'py str,
         mode: Option<PySplitMode>,
         logger: Option<PyObject>,
-    ) -> PyResult<PyMorphemeListWrapper> {
+        out: Option<&'py PyCell<PyMorphemeListWrapper>>,
+    ) -> PyResult<&'py PyCell<PyMorphemeListWrapper>> {
         // keep default mode to restore later
         let default_mode = mode.map(|m| self.tokenizer.set_mode(m.into()));
 
         self.tokenizer.reset().push_str(text);
-        self.tokenizer.do_tokenize().map_err(|e| {
-            PyException::new_err(format!("Error while tokenization: {}", e.to_string()))
-        })?;
+        self.tokenizer
+            .do_tokenize()
+            .map_err(|e| PyException::new_err(format!("Tokenization error: {}", e.to_string())))?;
+
+        let out_list = match out {
+            None => {
+                let morphemes = MorphemeList::empty(self.tokenizer.dict_clone());
+                let wrapper = PyMorphemeListWrapper::from(morphemes);
+                PyCell::new(py, wrapper)?
+            }
+            Some(list) => list,
+        };
 
-        let mut morphemes = MorphemeList::empty(self.tokenizer.dict_clone());
+        let mut borrow = out_list.try_borrow_mut();
+        let morphemes = match borrow {
+            Ok(ref mut ms) => ms.internal_mut(py),
+            Err(e) => return Err(PyException::new_err("out was used twice at the same time")),
+        };
 
         morphemes
             .collect_results(&mut self.tokenizer)
-            .map_err(|e| {
-                PyException::new_err(format!("Error while tokenization: {}", e.to_string()))
-            })?;
+            .map_err(|e| PyException::new_err(format!("Tokenization error: {}", e.to_string())))?;
 
         // restore default mode
         default_mode.map(|m| self.tokenizer.set_mode(m));
 
-        let wrapper = PyMorphemeListWrapper::from(morphemes);
-
-        Ok(wrapper)
+        Ok(out_list)
     }
 }
diff --git a/sudachi-cli/src/output.rs b/sudachi-cli/src/output.rs
index bafbfb23..d242ce53 100644
--- a/sudachi-cli/src/output.rs
+++ b/sudachi-cli/src/output.rs
@@ -109,7 +109,7 @@ fn write_morpheme_basic<T: DictionaryAccess>(
 ) -> SudachiResult<()> {
     writer.write_all(morpheme.surface().as_bytes())?;
     writer.write_all(b"\t")?;
-    let all_pos = morpheme.part_of_speech()?;
+    let all_pos = morpheme.part_of_speech();
     for (idx, pos) in all_pos.iter().enumerate() {
         writer.write_all(pos.as_bytes())?;
         if idx + 1 != all_pos.len() {
diff --git a/sudachi/src/analysis/mlist.rs b/sudachi/src/analysis/mlist.rs
new file mode 100644
index 00000000..4b33f376
--- /dev/null
+++ b/sudachi/src/analysis/mlist.rs
@@ -0,0 +1,252 @@
+/*
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+use crate::analysis::morpheme::Morpheme;
+use crate::analysis::node::{PathCost, ResultNode};
+use crate::analysis::stateful_tokenizer::StatefulTokenizer;
+use crate::analysis::stateless_tokenizer::DictionaryAccess;
+use crate::analysis::Mode;
+use crate::dic::subset::InfoSubset;
+use crate::error::{SudachiError, SudachiResult};
+use crate::input_text::InputBuffer;
+use std::cell::{Ref, RefCell};
+use std::iter::FusedIterator;
+use std::ops::{Deref, DerefMut, Index};
+use std::rc::Rc;
+
+struct InputPart {
+    input: InputBuffer,
+    subset: InfoSubset,
+}
+
+impl Default for InputPart {
+    fn default() -> Self {
+        let mut input = InputBuffer::new();
+        input.start_build().unwrap();
+        Self {
+            input,
+            subset: Default::default(),
+        }
+    }
+}
+
+#[derive(Default)]
+struct Nodes {
+    data: Vec<ResultNode>,
+}
+
+impl Nodes {
+    fn mut_data(&mut self) -> &mut Vec<ResultNode> {
+        &mut self.data
+    }
+}
+
+pub struct MorphemeList<T> {
+    dict: T,
+    input: Rc<RefCell<InputPart>>,
+    nodes: Nodes,
+}
+
+impl<T: DictionaryAccess> MorphemeList<T> {
+    /// Returns an empty morpheme list
+    pub fn empty(dict: T) -> Self {
+        let input = Default::default();
+        Self {
+            dict,
+            input: Rc::new(RefCell::new(input)),
+            nodes: Default::default(),
+        }
+    }
+
+    /// Creates MorphemeList from components
+    pub fn from_components(
+        dict: T,
+        input: InputBuffer,
+        path: Vec<ResultNode>,
+        subset: InfoSubset,
+    ) -> Self {
+        let input = InputPart { input, subset };
+        Self {
+            dict,
+            input: Rc::new(RefCell::new(input)),
+            nodes: Nodes { data: path },
+        }
+    }
+
+    pub fn collect_results<U: DictionaryAccess>(
+        &mut self,
+        analyzer: &mut StatefulTokenizer<U>,
+    ) -> SudachiResult<()> {
+        match self.input.try_borrow_mut() {
+            Ok(mut i) => {
+                let mref = i.deref_mut();
+                analyzer.swap_result(
+                    &mut mref.input,
+                    &mut self.nodes.mut_data(),
+                    &mut mref.subset,
+                );
+                Ok(())
+            }
+            Err(_) => Err(SudachiError::MorphemeListBorrowed),
+        }
+    }
+
+    /// Splits morphemes and writes them into the resulting list
+    /// The resulting list is _not_ cleared before that
+    /// Returns true if split produced more than two elements
+    pub fn split_into(&self, mode: Mode, index: usize, out: &mut Self) -> SudachiResult<bool> {
+        let node = self.node(index);
+        let num_splits = node.num_splits(mode);
+
+        if num_splits == 0 {
+            Ok(false)
+        } else {
+            out.assign_input(self);
+            let data = out.nodes.mut_data();
+            let input = self.input();
+            let subset = self.subset();
+            data.reserve(num_splits);
+            for n in node.split(mode, self.dict().lexicon(), subset, input.deref()) {
+                data.push(n);
+            }
+            Ok(true)
+        }
+    }
+
+    /// Clears morphemes from analysis result
+    pub fn clear(&mut self) {
+        self.nodes.mut_data().clear();
+    }
+
+    pub fn len(&self) -> usize {
+        self.nodes.data.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.nodes.data.is_empty()
+    }
+
+    pub fn get(&self, idx: usize) -> Morpheme<T> {
+        return Morpheme::for_list(self, idx);
+    }
+
+    pub fn surface(&self) -> Ref<str> {
+        let inp = self.input();
+        Ref::map(inp, |i| i.current())
+    }
+
+    pub fn iter(&self) -> MorphemeIter<T> {
+        MorphemeIter {
+            index: 0,
+            list: self,
+        }
+    }
+
+    /// Gets the whole cost of the path
+    pub fn get_internal_cost(&self) -> i32 {
+        let len = self.len();
+        if len == 0 {
+            return 0;
+        }
+
+        let first_node = self.node(0);
+        let last_node = self.node(len - 1);
+        last_node.total_cost() - first_node.total_cost()
+    }
+
+    pub(crate) fn node(&self, idx: usize) -> &ResultNode {
+        self.nodes.data.index(idx)
+    }
+
+    pub fn dict(&self) -> &T {
+        &self.dict
+    }
+
+    pub(crate) fn input(&self) -> Ref<InputBuffer> {
+        Ref::map(self.input.deref().borrow(), |x| &x.input)
+    }
+
+    /// Makes this point to the input of another MorphemeList
+    pub(crate) fn assign_input(&mut self, other: &Self) {
+        if self.input.as_ptr() != other.input.as_ptr() {
+            self.input = other.input.clone();
+        }
+    }
+
+    pub fn subset(&self) -> InfoSubset {
+        self.input.deref().borrow().subset
+    }
+
+    pub fn copy_slice(&self, start: usize, end: usize, out: &mut Self) {
+        let out_data = out.nodes.mut_data();
+        out_data.extend_from_slice(&self.nodes.data[start..end]);
+    }
+}
+
+impl<T: DictionaryAccess + Clone> MorphemeList<T> {
+    pub fn empty_clone(&self) -> Self {
+        Self {
+            dict: self.dict.clone(),
+            input: self.input.clone(),
+            nodes: Default::default(),
+        }
+    }
+
+    /// Returns a new morpheme list splitting the morpheme with a given mode.
+    /// Returns an empty list if there was no splits
+    #[deprecated(note = "use split_into", since = "0.6.1")]
+    pub fn split(&self, mode: Mode, index: usize) -> SudachiResult<MorphemeList<T>> {
+        let mut list = self.empty_clone();
+        if !self.split_into(mode, index, &mut list)? {
+            list.nodes.mut_data().push(self.node(index).clone())
+        }
+        Ok(list)
+    }
+}
+
+/// Iterates over morpheme list
+pub struct MorphemeIter<'a, T> {
+    list: &'a MorphemeList<T>,
+    index: usize,
+}
+
+impl<'a, T: DictionaryAccess> Iterator for MorphemeIter<'a, T> {
+    type Item = Morpheme<'a, T>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.index >= self.list.len() {
+            return None;
+        }
+
+        let morpheme = Morpheme::for_list(self.list, self.index);
+
+        self.index += 1;
+        Some(morpheme)
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let rem = self.list.len() - self.index;
+        (rem, Some(rem))
+    }
+}
+
+impl<'a, T: DictionaryAccess> FusedIterator for MorphemeIter<'a, T> {}
+
+impl<'a, T: DictionaryAccess> ExactSizeIterator for MorphemeIter<'a, T> {
+    fn len(&self) -> usize {
+        self.size_hint().0
+    }
+}
diff --git a/sudachi/src/analysis/mod.rs b/sudachi/src/analysis/mod.rs
index 1f58e078..7dd4046c 100644
--- a/sudachi/src/analysis/mod.rs
+++ b/sudachi/src/analysis/mod.rs
@@ -17,12 +17,13 @@
 use std::fmt::{Display, Formatter};
 use std::str::FromStr;
 
-use morpheme::MorphemeList;
+use mlist::MorphemeList;
 
 use crate::error::SudachiResult;
 
 mod inner;
 pub mod lattice;
+pub mod mlist;
 pub mod morpheme;
 pub mod node;
 pub mod stateful_tokenizer;
diff --git a/sudachi/src/analysis/morpheme.rs b/sudachi/src/analysis/morpheme.rs
index 6dd28025..c412e66b 100644
--- a/sudachi/src/analysis/morpheme.rs
+++ b/sudachi/src/analysis/morpheme.rs
@@ -15,197 +15,12 @@
  */
 
 use crate::analysis::node::{LatticeNode, PathCost, ResultNode};
-use crate::analysis::stateful_tokenizer::StatefulTokenizer;
 use crate::analysis::stateless_tokenizer::DictionaryAccess;
-use crate::dic::grammar::Grammar;
 use crate::dic::lexicon::word_infos::WordInfo;
-use crate::dic::subset::InfoSubset;
 use crate::dic::word_id::WordId;
-use crate::input_text::{InputBuffer, InputTextIndex};
+use crate::input_text::InputTextIndex;
 use crate::prelude::*;
-use std::sync::Arc;
-
-/// A list of morphemes
-#[derive(Clone)]
-pub struct MorphemeList<T> {
-    dict: T,
-    input_text: Arc<InputBuffer>,
-    path: Vec<ResultNode>,
-    subset: InfoSubset,
-}
-
-impl<T: DictionaryAccess> MorphemeList<T> {
-    pub fn from_components(
-        dict: T,
-        input: InputBuffer,
-        path: Vec<ResultNode>,
-        subset: InfoSubset,
-    ) -> SudachiResult<Self> {
-        let list = Self {
-            dict,
-            input_text: Arc::new(input),
-            path,
-            subset,
-        };
-        Ok(list)
-    }
-
-    /// Returns an empty morpheme list.
-    pub fn empty(dict: T) -> Self {
-        let mut ml = InputBuffer::new();
-        ml.start_build().unwrap(); // will not fail
-        Self {
-            dict,
-            input_text: Arc::new(ml),
-            path: Default::default(),
-            subset: Default::default(),
-        }
-    }
-
-    pub fn collect_results<U: DictionaryAccess>(
-        &mut self,
-        analyzer: &mut StatefulTokenizer<U>,
-    ) -> SudachiResult<()> {
-        // make a copy of input buffer if something else exists
-        let itext = Arc::make_mut(&mut self.input_text);
-        analyzer.swap_result(itext, &mut self.path, &mut self.subset);
-        Ok(())
-    }
-
-    pub fn get_grammar(&self) -> &Grammar {
-        self.dict.grammar()
-    }
-}
-
-impl<T: DictionaryAccess + Clone> MorphemeList<T> {
-    /// Returns a new morpheme list splitting the morpheme with a given mode.
-    pub fn split(&self, mode: Mode, index: usize) -> SudachiResult<MorphemeList<T>> {
-        let node = &self.path[index];
-        let num_splits = node.num_splits(mode);
-
-        let path = if num_splits <= 1 {
-            vec![node.clone()]
-        } else {
-            node.split(mode, self.dict.lexicon(), self.subset, &self.input_text)
-                .collect()
-        };
-
-        Ok(MorphemeList {
-            dict: self.dict.clone(),
-            input_text: self.input_text.clone(),
-            path,
-            subset: self.subset,
-        })
-    }
-}
-
-impl<T> MorphemeList<T> {
-    pub fn iter(&self) -> MorphemeIter<T> {
-        MorphemeIter {
-            list: &self,
-            index: 0,
-        }
-    }
-
-    pub fn surface(&self) -> &str {
-        if self.len() == 0 {
-            return self.input_text.original();
-        }
-
-        // can be a slice of another list
-        let begin = self.path.first().unwrap().begin_bytes();
-        let end = self.path.last().unwrap().end_bytes();
-        self.input_text.orig_slice(begin..end)
-    }
-
-    pub fn get_node(&self, index: usize) -> &ResultNode {
-        &self.path[index]
-    }
-
-    pub fn get_begin(&self, index: usize) -> usize {
-        let mod_bytes = self.path[index].begin();
-        self.input_text.to_orig_byte_idx(mod_bytes)
-    }
-
-    pub fn get_end(&self, index: usize) -> usize {
-        let mod_bytes = self.path[index].end();
-        self.input_text.to_orig_byte_idx(mod_bytes)
-    }
-
-    /// Returns begin codepoint index in the original sentence
-    pub fn get_begin_cp(&self, index: usize) -> usize {
-        let begin_cp = self.path[index].begin();
-        self.input_text.to_orig_char_idx(begin_cp)
-    }
-
-    /// Returns end codepoint index in the original sentence
-    pub fn get_end_cp(&self, index: usize) -> usize {
-        let end_cp = self.path[index].end();
-        self.input_text.to_orig_char_idx(end_cp)
-    }
-
-    /// Returns a substring of the original text which corresponds to the morpheme
-    pub fn get_surface(&self, index: usize) -> &str {
-        let node = &self.path[index];
-        &self.input_text.orig_slice(node.bytes_range())
-    }
-
-    pub fn get_word_info(&self, index: usize) -> &WordInfo {
-        self.path[index].word_info()
-    }
-
-    pub fn is_oov(&self, index: usize) -> bool {
-        self.path[index].word_id().is_oov()
-    }
-
-    /// Returns the total cost of the path
-    pub fn get_internal_cost(&self) -> i32 {
-        if self.len() == 0 {
-            return 0;
-        }
-
-        let first = &self.path[0];
-        let last = self.path.last().unwrap();
-        last.total_cost() - first.total_cost()
-    }
-
-    pub fn len(&self) -> usize {
-        self.path.len()
-    }
-
-    pub fn get(&self, index: usize) -> Morpheme<T> {
-        debug_assert!(index < self.path.len());
-        Morpheme { list: self, index }
-    }
-
-    pub fn dic(&self) -> &T {
-        &self.dict
-    }
-}
-
-/// Iterates over morpheme list
-pub struct MorphemeIter<'a, T> {
-    list: &'a MorphemeList<T>,
-    index: usize,
-}
-
-impl<'a, T: DictionaryAccess> Iterator for MorphemeIter<'a, T> {
-    type Item = Morpheme<'a, T>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if let None = self.list.path.get(self.index) {
-            return None;
-        }
-
-        let morpheme = Morpheme {
-            list: self.list,
-            index: self.index,
-        };
-
-        self.index += 1;
-        Some(morpheme)
-    }
-}
+use std::cell::Ref;
 
 /// A morpheme (basic semantic unit of language)
 pub struct Morpheme<'a, T> {
@@ -215,43 +30,61 @@ pub struct Morpheme<'a, T> {
 
 impl<T: DictionaryAccess> Morpheme<'_, T> {
     /// Returns the part of speech
-    pub fn part_of_speech(&self) -> SudachiResult<&[String]> {
-        let pos_id = self.part_of_speech_id();
-        let pos = self
-            .list
-            .get_grammar()
-            .pos_list
-            .get(pos_id as usize)
-            .ok_or(SudachiError::MissingPartOfSpeech)?;
-        Ok(pos)
+    pub fn part_of_speech(&self) -> &[String] {
+        self.list
+            .dict()
+            .grammar()
+            .pos_components(self.part_of_speech_id())
     }
 }
 
 impl<T: DictionaryAccess + Clone> Morpheme<'_, T> {
     /// Returns new morpheme list splitting the morpheme with given mode.
+    #[deprecated(note = "use split_into", since = "0.6.1")]
     pub fn split(&self, mode: Mode) -> SudachiResult<MorphemeList<T>> {
+        #[allow(deprecated)]
         self.list.split(mode, self.index)
     }
 }
 
-impl<T> Morpheme<'_, T> {
+impl<'a, T: DictionaryAccess> Morpheme<'a, T> {
+    pub(crate) fn for_list(list: &'a MorphemeList<T>, index: usize) -> Self {
+        Morpheme { list, index }
+    }
+
+    #[inline]
+    pub(crate) fn node(&self) -> &ResultNode {
+        self.list.node(self.index)
+    }
+
     /// Returns the begin index in bytes of the morpheme in the original text
     pub fn begin(&self) -> usize {
-        self.list.get_begin(self.index)
+        self.list.input().to_orig_byte_idx(self.node().begin())
     }
 
     /// Returns the end index in bytes of the morpheme in the original text
     pub fn end(&self) -> usize {
-        self.list.get_end(self.index)
+        self.list.input().to_orig_byte_idx(self.node().end())
+    }
+
+    /// Returns the codepoint offset of the morpheme begin in the original text
+    pub fn begin_c(&self) -> usize {
+        self.list.input().to_orig_char_idx(self.node().begin())
+    }
+
+    /// Returns the codepoint offset of the morpheme begin in the original text
+    pub fn end_c(&self) -> usize {
+        self.list.input().to_orig_char_idx(self.node().end())
     }
 
     /// Returns a substring of the original text which corresponds to the morpheme
-    pub fn surface(&self) -> &str {
-        self.list.get_surface(self.index)
+    pub fn surface(&self) -> Ref<str> {
+        let inp = self.list.input();
+        Ref::map(inp, |i| i.orig_slice(self.node().bytes_range()))
     }
 
     pub fn part_of_speech_id(&self) -> u16 {
-        self.get_word_info().pos_id()
+        self.node().word_info().pos_id()
     }
 
     /// Returns the dictionary form of morpheme
@@ -277,12 +110,12 @@ impl<T> Morpheme<'_, T> {
 
     /// Returns if this morpheme is out of vocabulary
     pub fn is_oov(&self) -> bool {
-        self.list.is_oov(self.index)
+        self.word_id().is_oov()
     }
 
     /// Returns the word id of morpheme
     pub fn word_id(&self) -> WordId {
-        self.list.get_node(self.index).word_id()
+        self.node().word_id()
     }
 
     /// Returns the dictionary id where the morpheme belongs
@@ -302,12 +135,25 @@ impl<T> Morpheme<'_, T> {
     }
 
     pub fn get_word_info(&self) -> &WordInfo {
-        self.list.get_word_info(self.index)
+        self.node().word_info()
     }
 
+    /// Returns the index of this morpheme
     pub fn index(&self) -> usize {
         self.index
     }
+
+    /// Splits morpheme and writes sub-morphemes into the provided list.
+    /// The resulting list is _not_ cleared before that.
+    /// Returns true if split has produced any elements.
+    pub fn split_into(&self, mode: Mode, out: &mut MorphemeList<T>) -> SudachiResult<bool> {
+        self.list.split_into(mode, self.index, out)
+    }
+
+    /// Returns total cost from the beginning of the path
+    pub fn total_cost(&self) -> i32 {
+        return self.node().total_cost();
+    }
 }
 
 impl<T: DictionaryAccess> std::fmt::Debug for Morpheme<'_, T> {
diff --git a/sudachi/src/analysis/stateful_tokenizer.rs b/sudachi/src/analysis/stateful_tokenizer.rs
index 0147277f..b00b5045 100644
--- a/sudachi/src/analysis/stateful_tokenizer.rs
+++ b/sudachi/src/analysis/stateful_tokenizer.rs
@@ -283,9 +283,12 @@ impl<D: DictionaryAccess> StatefulTokenizer<D> {
     pub fn into_morpheme_list(self) -> SudachiResult<MorphemeList<D>> {
         match self.top_path {
             None => Err(SudachiError::EosBosDisconnect),
-            Some(path) => {
-                MorphemeList::from_components(self.dictionary, self.input, path, self.subset)
-            }
+            Some(path) => Ok(MorphemeList::from_components(
+                self.dictionary,
+                self.input,
+                path,
+                self.subset,
+            )),
         }
     }
 }
diff --git a/sudachi/src/analysis/stateless_tokenizer.rs b/sudachi/src/analysis/stateless_tokenizer.rs
index 0bd54c6f..5670d217 100644
--- a/sudachi/src/analysis/stateless_tokenizer.rs
+++ b/sudachi/src/analysis/stateless_tokenizer.rs
@@ -27,7 +27,7 @@ use crate::plugin::input_text::InputTextPlugin;
 use crate::plugin::oov::OovProviderPlugin;
 use crate::plugin::path_rewrite::PathRewritePlugin;
 
-use super::morpheme::MorphemeList;
+use super::mlist::MorphemeList;
 use super::{Mode, Tokenize};
 
 /// Provides access to dictionary data
diff --git a/sudachi/src/dic/grammar.rs b/sudachi/src/dic/grammar.rs
index 050359b1..f958a761 100644
--- a/sudachi/src/dic/grammar.rs
+++ b/sudachi/src/dic/grammar.rs
@@ -24,6 +24,7 @@ use nom::{
     bytes::complete::take,
     number::complete::{le_i16, le_u16},
 };
+use std::ops::Index;
 
 /// Dictionary grammar
 ///
@@ -117,6 +118,12 @@ impl<'a> Grammar<'a> {
         None
     }
 
+    /// Gets POS components for POS ID.
+    /// Panics if out of bounds.
+    pub fn pos_components(&self, pos_id: u16) -> &[String] {
+        self.pos_list.index(pos_id as usize)
+    }
+
     /// Merge a another grammar into this grammar
     ///
     /// Only pos_list is merged
diff --git a/sudachi/src/error.rs b/sudachi/src/error.rs
index 6a18d393..c22c02fa 100644
--- a/sudachi/src/error.rs
+++ b/sudachi/src/error.rs
@@ -110,6 +110,9 @@ pub enum SudachiError {
 
     #[error(transparent)]
     DictionaryCompilationError(#[from] DicBuildError),
+
+    #[error("MorphemeList is borrowed, make sure that all Ref<> are dropped")]
+    MorphemeListBorrowed,
 }
 
 pub type SudachiNomResult<I, O> = nom::IResult<I, O, SudachiNomError<I>>;
diff --git a/sudachi/src/lib.rs b/sudachi/src/lib.rs
index d4c92696..17a63ae2 100644
--- a/sudachi/src/lib.rs
+++ b/sudachi/src/lib.rs
@@ -40,9 +40,7 @@ pub mod test;
 
 pub mod prelude {
     pub use crate::{
-        analysis::morpheme::MorphemeList,
-        analysis::{Mode, Tokenize},
-        error::SudachiError,
-        error::SudachiResult,
+        analysis::mlist::MorphemeList, analysis::morpheme::Morpheme, analysis::Mode,
+        error::SudachiError, error::SudachiResult,
     };
 }


Test patch:
diff --git a/python/tests/test_morpheme.py b/python/tests/test_morpheme.py
index 063fd6e2..243a634c 100644
--- a/python/tests/test_morpheme.py
+++ b/python/tests/test_morpheme.py
@@ -132,6 +132,22 @@ def test_normalize_katakana_half_full(self):
         self.assertEqual(m[1].begin(), 2)
         self.assertEqual(len(m[0]), 2)
 
+    def test_morpheme_split_out(self):
+        ms = self.tokenizer_obj.tokenize('東京都', SplitMode.C)
+        self.assertEqual(1, ms.size())
+        self.assertEqual(ms[0].surface(), '東京都')
+
+        ms_a = ms[0].split(SplitMode.A, out=None)
+        self.assertEqual(2, ms_a.size())
+        self.assertEqual(ms_a[0].surface(), '東京')
+        self.assertEqual(ms_a[1].surface(), '都')
+
+        ms = self.tokenizer_obj.tokenize("京都東京都京都", SplitMode.C)
+        ms_b = ms[1].split(SplitMode.A, out=ms_a)
+        self.assertEqual(id(ms_a), id(ms_b))
+        self.assertEqual(ms_a[0].surface(), '東京')
+        self.assertEqual(ms_a[1].surface(), '都')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/python/tests/test_tokenizer.py b/python/tests/test_tokenizer.py
index 574157de..d30c82f9 100644
--- a/python/tests/test_tokenizer.py
+++ b/python/tests/test_tokenizer.py
@@ -130,6 +130,15 @@ def test_tokenizer_subset(self):
         ms2 = tok.tokenize('東京都')
         self.assertEqual(ms1[0].part_of_speech_id(), ms2[0].part_of_speech_id())
 
+    def test_tokenizer_out_param(self):
+        ms1 = self.tokenizer_obj.tokenize('東京都東京府')
+        m = ms1[0]
+        self.assertEqual(m.surface(), '東京都')
+
+        ms2 = self.tokenizer_obj.tokenize('すだち', out=ms1)
+        self.assertEqual(id(ms1), id(ms2))
+        self.assertEqual(m.surface(), 'すだち')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/sudachi/src/dic/build/test/with_analysis.rs b/sudachi/src/dic/build/test/with_analysis.rs
index 9516ef94..18f94a4d 100644
--- a/sudachi/src/dic/build/test/with_analysis.rs
+++ b/sudachi/src/dic/build/test/with_analysis.rs
@@ -172,7 +172,7 @@ fn system_plus_user_2() {
     let result = tok.tokenize("かぼすにいく", Mode::C, false).unwrap();
     assert_eq!(result.len(), 3);
     assert_eq!(result.get(0).word_id().dic(), 2);
-    assert_eq!(result.get(0).part_of_speech().unwrap()[0], "被子植物門");
+    assert_eq!(result.get(0).part_of_speech()[0], "被子植物門");
 }
 
 #[test]
@@ -193,5 +193,5 @@ fn split_with_subset() {
     res.collect_results(&mut tok).unwrap();
     assert_eq!(res.len(), 2);
     //assert_eq!(res.get_end(0), 6);
-    assert_eq!(res.get_end(1), 9);
+    assert_eq!(res.get(1).end(), 9);
 }
diff --git a/sudachi/tests/common/mod.rs b/sudachi/tests/common/mod.rs
index 8c13bc0b..b2fdf1ef 100644
--- a/sudachi/tests/common/mod.rs
+++ b/sudachi/tests/common/mod.rs
@@ -31,6 +31,7 @@ use sudachi::dic::{grammar::Grammar, header::Header, lexicon::Lexicon, Dictionar
 use sudachi::prelude::*;
 
 use lazy_static::lazy_static;
+use sudachi::analysis::Tokenize;
 use sudachi::dic::build::DictBuilder;
 use sudachi::dic::storage::{Storage, SudachiDicData};
 
diff --git a/sudachi/tests/morpheme.rs b/sudachi/tests/morpheme.rs
index 3b1e3dde..c13624d4 100644
--- a/sudachi/tests/morpheme.rs
+++ b/sudachi/tests/morpheme.rs
@@ -17,8 +17,8 @@
 extern crate lazy_static;
 extern crate sudachi;
 
-use sudachi::analysis::morpheme::MorphemeList;
-use sudachi::prelude::Mode;
+use std::ops::Deref;
+use sudachi::prelude::*;
 
 mod common;
 use crate::common::TestTokenizer;
@@ -28,7 +28,7 @@ fn empty_morpheme_list() {
     let tok = TestTokenizer::new();
     let empty = MorphemeList::empty(tok.dict());
 
-    assert_eq!("", empty.surface());
+    assert_eq!("", empty.surface().deref());
     assert_eq!(0, empty.len());
 }
 
@@ -36,25 +36,26 @@ fn empty_morpheme_list() {
 fn morpheme_attributes() {
     let tok = TestTokenizer::new();
     let ms = tok.tokenize("京都", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
 
-    assert_eq!(0, ms[0].begin());
-    assert_eq!(6, ms[0].end());
-    assert_eq!("京都", ms[0].surface());
+    assert_eq!(0, ms.get(0).begin());
+    assert_eq!(6, ms.get(0).end());
+    assert_eq!("京都", ms.get(0).surface().deref());
 
-    let pos = ms[0].part_of_speech().expect("failed to get pos");
-    assert_eq!(["名詞", "固有名詞", "地名", "一般", "*", "*"], &pos[..]);
-    assert_eq!(3, ms[0].part_of_speech_id());
+    assert_eq!(
+        ["名詞", "固有名詞", "地名", "一般", "*", "*"],
+        ms.get(0).part_of_speech()
+    );
+    assert_eq!(3, ms.get(0).part_of_speech_id());
 
-    assert_eq!("京都", ms[0].dictionary_form());
-    assert_eq!("京都", ms[0].normalized_form());
-    assert_eq!("キョウト", ms[0].reading_form());
+    assert_eq!("京都", ms.get(0).dictionary_form());
+    assert_eq!("京都", ms.get(0).normalized_form());
+    assert_eq!("キョウト", ms.get(0).reading_form());
 
-    assert_eq!(false, ms[0].is_oov());
+    assert_eq!(false, ms.get(0).is_oov());
 
-    assert_eq!(3, ms[0].word_id().word());
-    assert_eq!(0, ms[0].dictionary_id());
-    assert_eq!([1, 5], ms[0].synonym_group_ids());
+    assert_eq!(3, ms.get(0).word_id().word());
+    assert_eq!(0, ms.get(0).dictionary_id());
+    assert_eq!([1, 5], ms.get(0).synonym_group_ids());
 }
 
 #[test]
@@ -62,15 +63,14 @@ fn split_morpheme() {
     let tok = TestTokenizer::new();
     let ms = tok.tokenize("京都東京都", Mode::C);
     assert_eq!(2, ms.len());
-    let ms: Vec<_> = ms.iter().collect();
-    assert_eq!("京都", ms[0].surface());
-    assert_eq!("東京都", ms[1].surface());
+    assert_eq!("京都", ms.get(0).surface().deref());
+    assert_eq!("東京都", ms.get(1).surface().deref());
 
-    let ms = ms[1].split(Mode::A).expect("failed to split morpheme");
+    #[allow(deprecated)]
+    let ms = ms.get(1).split(Mode::A).expect("failed to split morpheme");
     assert_eq!(2, ms.len());
-    let ms: Vec<_> = ms.iter().collect();
-    assert_eq!("東京", ms[0].surface());
-    assert_eq!(6, ms[0].begin()); // keep index for the whole input text
-    assert_eq!(12, ms[0].end());
-    assert_eq!("都", ms[1].surface());
+    assert_eq!("東京", ms.get(0).surface().deref());
+    assert_eq!(6, ms.get(0).begin()); // keep index for the whole input text
+    assert_eq!(12, ms.get(0).end());
+    assert_eq!("都", ms.get(1).surface().deref());
 }
diff --git a/sudachi/tests/split.rs b/sudachi/tests/split.rs
index 426552a6..2c150a42 100644
--- a/sudachi/tests/split.rs
+++ b/sudachi/tests/split.rs
@@ -15,6 +15,7 @@
  */
 
 use crate::common::TestStatefulTokenizer;
+use std::ops::Deref;
 use sudachi::prelude::Mode;
 
 mod common;
@@ -26,8 +27,8 @@ fn split_at_analysis_time() {
     let mut tok = TestStatefulTokenizer::builder(AB_DIC).mode(Mode::A).build();
     let res = tok.tokenize("ＡＢ");
     assert_eq!(res.len(), 2);
-    assert_eq!(res.get(0).surface(), "Ａ");
-    assert_eq!(res.get(1).surface(), "Ｂ");
+    assert_eq!(res.get(0).surface().deref(), "Ａ");
+    assert_eq!(res.get(1).surface().deref(), "Ｂ");
 }
 
 #[test]
@@ -35,7 +36,8 @@ fn split_after_analysis() {
     let mut tok = TestStatefulTokenizer::builder(AB_DIC).build();
     let res = tok.tokenize("ＡＢ");
     assert_eq!(res.len(), 1);
-    let res2 = res.get(0).split(Mode::A).unwrap();
-    assert_eq!(res2.get(0).surface(), "Ａ");
-    assert_eq!(res2.get(1).surface(), "Ｂ");
+    let mut res2 = res.empty_clone();
+    assert!(res.get(0).split_into(Mode::A, &mut res2).unwrap());
+    assert_eq!(res2.get(0).surface().deref(), "Ａ");
+    assert_eq!(res2.get(1).surface().deref(), "Ｂ");
 }
diff --git a/sudachi/tests/stateful_tokenizer.rs b/sudachi/tests/stateful_tokenizer.rs
index ff00735d..482eb5f1 100644
--- a/sudachi/tests/stateful_tokenizer.rs
+++ b/sudachi/tests/stateful_tokenizer.rs
@@ -17,7 +17,7 @@
 extern crate lazy_static;
 extern crate sudachi;
 
-use sudachi::analysis::node::LatticeNode;
+use std::ops::Deref;
 use sudachi::prelude::Mode;
 
 mod common;
@@ -43,14 +43,14 @@ fn get_word_id() {
     let ms = tok.tokenize("京都");
     assert_eq!(1, ms.len());
     let m0 = ms.get(0);
-    let pos = m0.part_of_speech().expect("failed to get pos");
+    let pos = m0.part_of_speech();
     assert_eq!(&["名詞", "固有名詞", "地名", "一般", "*", "*"], pos);
 
     // we do not have word_id field in Morpheme and skip testing.
     let ms = tok.tokenize("ぴらる");
     assert_eq!(1, ms.len());
     let m0 = ms.get(0);
-    let pos = m0.part_of_speech().expect("failed to get pos");
+    let pos = m0.part_of_speech();
     assert_eq!(&["名詞", "普通名詞", "一般", "*", "*", "*"], pos);
 }
 
@@ -58,19 +58,16 @@ fn get_word_id() {
 fn get_dictionary_id() {
     let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("京都");
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert_eq!(0, ms[0].dictionary_id());
+    assert_eq!(0, ms.get(0).dictionary_id());
 
     let ms = tok.tokenize("ぴらる");
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert_eq!(1, ms[0].dictionary_id());
+    assert_eq!(1, ms.get(0).dictionary_id());
 
     let ms = tok.tokenize("京");
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert!(ms[0].dictionary_id() < 0);
+    assert!(ms.get(0).dictionary_id() < 0);
 }
 
 #[test]
@@ -102,11 +99,11 @@ fn tokenize_with_dots() {
     let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("京都…");
     assert_eq!(4, ms.len());
-    assert_eq!("…", ms.get(1).surface());
+    assert_eq!("…", ms.get(1).surface().deref());
     assert_eq!(".", ms.get(1).normalized_form());
-    assert_eq!("", ms.get(2).surface());
+    assert_eq!("", ms.get(2).surface().deref());
     assert_eq!(".", ms.get(2).normalized_form());
-    assert_eq!("", ms.get(3).surface());
+    assert_eq!("", ms.get(3).surface().deref());
     assert_eq!(".", ms.get(3).normalized_form());
 }
 
@@ -115,13 +112,13 @@ fn tokenizer_morpheme_split() {
     let mut tok = TestTokenizer::new_built(Mode::C);
     let ms = tok.tokenize("東京都");
     assert_eq!(1, ms.len());
-    assert_eq!("東京都", ms.get(0).surface());
+    assert_eq!("東京都", ms.get(0).surface().deref());
 
     tok.set_mode(Mode::A);
     let ms = tok.tokenize("東京都");
     assert_eq!(2, ms.len());
-    assert_eq!("東京", ms.get(0).surface());
-    assert_eq!("都", ms.get(1).surface());
+    assert_eq!("東京", ms.get(0).surface().deref());
+    assert_eq!("都", ms.get(1).surface().deref());
 }
 
 #[test]
@@ -130,18 +127,19 @@ fn split_middle() {
     let ms = tok.tokenize("京都東京都京都");
     assert_eq!(ms.len(), 3);
     let m = ms.get(1);
-    assert_eq!(m.surface(), "東京都");
+    assert_eq!(m.surface().deref(), "東京都");
 
-    let ms_a = m.split(Mode::A).expect("works");
+    let mut ms_a = ms.empty_clone();
+    assert!(m.split_into(Mode::A, &mut ms_a).expect("works"));
     assert_eq!(ms_a.len(), 2);
-    assert_eq!(ms_a.get(0).surface(), "東京");
-    assert_eq!(ms_a.get_node(0).begin(), 2);
-    assert_eq!(ms_a.get_node(0).end(), 4);
+    assert_eq!(ms_a.get(0).surface().deref(), "東京");
+    assert_eq!(ms_a.get(0).begin_c(), 2);
+    assert_eq!(ms_a.get(0).end_c(), 4);
     assert_eq!(ms_a.get(0).begin(), 6);
     assert_eq!(ms_a.get(0).end(), 12);
-    assert_eq!(ms_a.get(1).surface(), "都");
-    assert_eq!(ms_a.get_node(1).begin(), 4);
-    assert_eq!(ms_a.get_node(1).end(), 5);
+    assert_eq!(ms_a.get(1).surface().deref(), "都");
+    assert_eq!(ms_a.get(1).begin_c(), 4);
+    assert_eq!(ms_a.get(1).end_c(), 5);
     assert_eq!(ms_a.get(1).begin(), 12);
     assert_eq!(ms_a.get(1).end(), 15);
 }
diff --git a/sudachi/tests/stateless_tokenizer.rs b/sudachi/tests/stateless_tokenizer.rs
index 4b1d9044..e0d9f619 100644
--- a/sudachi/tests/stateless_tokenizer.rs
+++ b/sudachi/tests/stateless_tokenizer.rs
@@ -17,6 +17,7 @@
 extern crate lazy_static;
 extern crate sudachi;
 
+use std::ops::Deref;
 use sudachi::prelude::Mode;
 
 mod common;
@@ -33,55 +34,51 @@ fn tokenize_small_katakana_only() {
 fn get_word_id() {
     let tok = TestTokenizer::new();
     let ms = tok.tokenize("京都", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    let pos = ms[0].part_of_speech().expect("failed to get pos");
-    assert_eq!(["名詞", "固有名詞", "地名", "一般", "*", "*"], &pos[..]);
+    let m = ms.get(0);
+    let pos = m.part_of_speech();
+    assert_eq!(["名詞", "固有名詞", "地名", "一般", "*", "*"], pos);
 
     // we do not have word_id field in Morpheme and skip testing.
     let ms = tok.tokenize("ぴらる", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    let pos = ms[0].part_of_speech().expect("failed to get pos");
-    assert_eq!(["名詞", "普通名詞", "一般", "*", "*", "*"], &pos[..]);
+    let m = ms.get(0);
+    assert_eq!(
+        ["名詞", "普通名詞", "一般", "*", "*", "*"],
+        m.part_of_speech()
+    );
 }
 
 #[test]
 fn get_dictionary_id() {
     let tok = TestTokenizer::new();
     let ms = tok.tokenize("京都", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert_eq!(0, ms[0].dictionary_id());
+    assert_eq!(0, ms.get(0).dictionary_id());
 
     let ms = tok.tokenize("ぴらる", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert_eq!(1, ms[0].dictionary_id());
+    assert_eq!(1, ms.get(0).dictionary_id());
 
     let ms = tok.tokenize("京", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert!(ms[0].dictionary_id() < 0);
+    assert!(ms.get(0).dictionary_id() < 0);
 }
 
 #[test]
 fn get_synonym_group_id() {
     let tok = TestTokenizer::new();
     let ms = tok.tokenize("京都", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert_eq!([1, 5], ms[0].synonym_group_ids());
+    assert_eq!([1, 5], ms.get(0).synonym_group_ids());
 
     let ms = tok.tokenize("ぴらる", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert!(ms[0].synonym_group_ids().is_empty());
+    assert!(ms.get(0).synonym_group_ids().is_empty());
 
     let ms = tok.tokenize("東京府", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert_eq!([1, 3], ms[0].synonym_group_ids());
+    assert_eq!([1, 3], ms.get(0).synonym_group_ids());
 }
 
 #[test]
@@ -96,27 +93,24 @@ fn tokenize_kanji_alphabet_word() {
 fn tokenize_with_dots() {
     let tok = TestTokenizer::new();
     let ms = tok.tokenize("京都…", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(4, ms.len());
-    assert_eq!("…", ms[1].surface());
-    assert_eq!(".", ms[1].normalized_form());
-    assert_eq!("", ms[2].surface());
-    assert_eq!(".", ms[2].normalized_form());
-    assert_eq!("", ms[3].surface());
-    assert_eq!(".", ms[3].normalized_form());
+    assert_eq!("…", ms.get(1).surface().deref());
+    assert_eq!(".", ms.get(1).normalized_form());
+    assert_eq!("", ms.get(2).surface().deref());
+    assert_eq!(".", ms.get(2).normalized_form());
+    assert_eq!("", ms.get(3).surface().deref());
+    assert_eq!(".", ms.get(3).normalized_form());
 }
 
 #[test]
 fn tokenizer_morpheme_split() {
     let tok = TestTokenizer::new();
     let ms = tok.tokenize("東京都", Mode::C);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(1, ms.len());
-    assert_eq!("東京都", ms[0].surface());
+    assert_eq!("東京都", ms.get(0).surface().deref());
 
     let ms = tok.tokenize("東京都", Mode::A);
-    let ms: Vec<_> = ms.iter().collect();
     assert_eq!(2, ms.len());
-    assert_eq!("東京", ms[0].surface());
-    assert_eq!("都", ms[1].surface());
+    assert_eq!("東京", ms.get(0).surface().deref());
+    assert_eq!("都", ms.get(1).surface().deref());
 }