Repository: WorksApplications/sudachi.rs (rust)

Issue:
Move analyzer internals to analysis package
Right now some of them are in root, lattice is separated into subpackage.

Merge lattice and analyzer into a single package.

Pull request:
move analyzer internals to analysis package
fixes #61

Patch:
diff --git a/python/src/dictionary.rs b/python/src/dictionary.rs
index 873c34c..93a0af1 100644
--- a/python/src/dictionary.rs
+++ b/python/src/dictionary.rs
@@ -20,9 +20,9 @@ use std::sync::Arc;
 use pyo3::exceptions::PyException;
 use pyo3::prelude::*;
 
+use sudachi::analysis::stateless_tokenizer::StatelessTokenizer;
 use sudachi::config::Config;
 use sudachi::dic::dictionary::JapaneseDictionary;
-use sudachi::stateless_tokeniser::StatelessTokenizer;
 
 use crate::tokenizer::{PySplitMode, PyTokenizer};
 
diff --git a/python/src/tokenizer.rs b/python/src/tokenizer.rs
index 1183dcb..702ce0f 100644
--- a/python/src/tokenizer.rs
+++ b/python/src/tokenizer.rs
@@ -19,9 +19,9 @@ use std::sync::Arc;
 use pyo3::exceptions::PyException;
 use pyo3::prelude::*;
 
+use sudachi::analysis::stateless_tokenizer::StatelessTokenizer;
 use sudachi::dic::dictionary::JapaneseDictionary;
 use sudachi::prelude::*;
-use sudachi::stateless_tokeniser::StatelessTokenizer;
 
 use crate::morpheme::PyMorpheme;
 
diff --git a/sudachi-cli/src/main.rs b/sudachi-cli/src/main.rs
index fea036c..d43dba9 100644
--- a/sudachi-cli/src/main.rs
+++ b/sudachi-cli/src/main.rs
@@ -21,11 +21,11 @@ use std::process;
 
 use structopt::StructOpt;
 
+use sudachi::analysis::stateless_tokenizer::StatelessTokenizer;
 use sudachi::config::Config;
 use sudachi::dic::dictionary::JapaneseDictionary;
 use sudachi::prelude::*;
 use sudachi::sentence_splitter::{SentenceSplitter, SplitSentences};
-use sudachi::stateless_tokeniser::StatelessTokenizer;
 
 #[cfg(feature = "bake_dictionary")]
 const BAKED_DICTIONARY_BYTES: &[u8] = include_bytes!(env!("SUDACHI_DICT_PATH"));
diff --git a/sudachi/src/lattice/mod.rs b/sudachi/src/analysis/lattice.rs
similarity index 88%
rename from sudachi/src/lattice/mod.rs
rename to sudachi/src/analysis/lattice.rs
index 6face10..bd7667d 100644
--- a/sudachi/src/lattice/mod.rs
+++ b/sudachi/src/analysis/lattice.rs
@@ -1,27 +1,23 @@
 /*
- * Copyright (c) 2021 Works Applications Co., Ltd.
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 
-use std::i32;
-
-use self::node::Node;
+use super::node::Node;
 use crate::dic::grammar::Grammar;
 use crate::dic::lexicon_set::LexiconSet;
-use crate::prelude::*;
-
-pub mod node;
+use crate::error::{SudachiError, SudachiResult};
 
 /// Lattice for tokenization
 pub struct Lattice<'a> {
diff --git a/sudachi/src/tokenizer.rs b/sudachi/src/analysis/mod.rs
similarity index 93%
rename from sudachi/src/tokenizer.rs
rename to sudachi/src/analysis/mod.rs
index 07b86e8..44ea3dd 100644
--- a/sudachi/src/tokenizer.rs
+++ b/sudachi/src/analysis/mod.rs
@@ -16,23 +16,14 @@
 
 use std::str::FromStr;
 
-use crate::morpheme::Morpheme;
-use crate::prelude::*;
+use morpheme::Morpheme;
 
-/// Able to tokenize Japanese text
-pub trait Tokenize {
-    /// Break text into `Morpheme`s
-    fn tokenize(&self, input: &str, mode: Mode, enable_debug: bool)
-        -> SudachiResult<Vec<Morpheme>>;
+use crate::error::SudachiResult;
 
-    /// Split text into sentences then tokenize
-    fn tokenize_sentences(
-        &self,
-        input: &str,
-        mode: Mode,
-        enable_debug: bool,
-    ) -> SudachiResult<Vec<Vec<Morpheme>>>;
-}
+pub mod lattice;
+pub mod morpheme;
+pub mod node;
+pub mod stateless_tokenizer;
 
 /// Unit to split text
 ///
@@ -81,3 +72,18 @@ impl FromStr for Mode {
         }
     }
 }
+
+/// Able to tokenize Japanese text
+pub trait Tokenize {
+    /// Break text into `Morpheme`s
+    fn tokenize(&self, input: &str, mode: Mode, enable_debug: bool)
+        -> SudachiResult<Vec<Morpheme>>;
+
+    /// Split text into sentences then tokenize
+    fn tokenize_sentences(
+        &self,
+        input: &str,
+        mode: Mode,
+        enable_debug: bool,
+    ) -> SudachiResult<Vec<Vec<Morpheme>>>;
+}
diff --git a/sudachi/src/morpheme.rs b/sudachi/src/analysis/morpheme.rs
similarity index 81%
rename from sudachi/src/morpheme.rs
rename to sudachi/src/analysis/morpheme.rs
index 81135ef..c651a11 100644
--- a/sudachi/src/morpheme.rs
+++ b/sudachi/src/analysis/morpheme.rs
@@ -1,26 +1,26 @@
 /*
- * Copyright (c) 2021 Works Applications Co., Ltd.
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 
 use std::fmt;
 
+use crate::analysis::node::Node;
 use crate::dic::grammar::Grammar;
 use crate::dic::lexicon::word_infos::WordInfo;
 use crate::dic::lexicon_set::LexiconSet;
 use crate::input_text::Utf8InputText;
-use crate::lattice::node::Node;
 use crate::prelude::*;
 
 /// A morpheme (basic semantic unit of language)
diff --git a/sudachi/src/lattice/node.rs b/sudachi/src/analysis/node.rs
similarity index 100%
rename from sudachi/src/lattice/node.rs
rename to sudachi/src/analysis/node.rs
diff --git a/sudachi/src/stateless_tokeniser.rs b/sudachi/src/analysis/stateless_tokenizer.rs
similarity index 91%
rename from sudachi/src/stateless_tokeniser.rs
rename to sudachi/src/analysis/stateless_tokenizer.rs
index 59c68ce..daab232 100644
--- a/sudachi/src/stateless_tokeniser.rs
+++ b/sudachi/src/analysis/stateless_tokenizer.rs
@@ -1,17 +1,17 @@
 /*
- * Copyright (c) 2021 Works Applications Co., Ltd.
+ *  Copyright (c) 2021 Works Applications Co., Ltd.
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *   Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 
 use std::ops::Deref;
@@ -21,15 +21,16 @@ use crate::dic::grammar::Grammar;
 use crate::dic::lexicon_set::LexiconSet;
 use crate::error::{SudachiError, SudachiResult};
 use crate::input_text::{Utf8InputText, Utf8InputTextBuilder};
-use crate::lattice::node::Node;
-use crate::lattice::Lattice;
-use crate::morpheme::Morpheme;
 use crate::plugin::input_text::InputTextPlugin;
 use crate::plugin::oov::OovProviderPlugin;
 use crate::plugin::path_rewrite::PathRewritePlugin;
-use crate::prelude::{Mode, Tokenize};
 use crate::sentence_detector::{NonBreakChecker, SentenceDetector};
 
+use super::lattice::Lattice;
+use super::morpheme::Morpheme;
+use super::node::Node;
+use super::{Mode, Tokenize};
+
 /// Provides access to dictionary data
 pub trait DictionaryAccess {
     fn grammar(&self) -> &Grammar<'_>;
diff --git a/sudachi/src/dic/dictionary.rs b/sudachi/src/dic/dictionary.rs
index b8abeee..54de6b2 100644
--- a/sudachi/src/dic/dictionary.rs
+++ b/sudachi/src/dic/dictionary.rs
@@ -19,6 +19,7 @@ use std::path::Path;
 
 use memmap2::Mmap;
 
+use crate::analysis::stateless_tokenizer::{DictionaryAccess, StatelessTokenizer};
 use crate::config::Config;
 use crate::config::ConfigError::MissingArgument;
 use crate::dic::grammar::Grammar;
@@ -30,7 +31,6 @@ use crate::plugin::input_text::InputTextPlugin;
 use crate::plugin::oov::OovProviderPlugin;
 use crate::plugin::path_rewrite::PathRewritePlugin;
 use crate::plugin::Plugins;
-use crate::stateless_tokeniser::{DictionaryAccess, StatelessTokenizer};
 
 // Mmap stores file handle, so file field may not be needed
 // but let it be here for the time being
diff --git a/sudachi/src/lib.rs b/sudachi/src/lib.rs
index 3528bc7..8e0aee7 100644
--- a/sudachi/src/lib.rs
+++ b/sudachi/src/lib.rs
@@ -21,23 +21,20 @@
 //! [`Tokenizer`](tokenizer/struct.Tokenizer.html) struct, which
 //! implements [`Tokenize`](tokenizer/trait.Tokenize.html).
 
+pub mod analysis;
 pub mod config;
 pub mod dic;
 pub mod error;
 pub mod input_text;
-pub mod lattice;
-pub mod morpheme;
 pub mod plugin;
 pub mod sentence_detector;
 pub mod sentence_splitter;
-pub mod stateless_tokeniser;
-pub mod tokenizer;
 
 pub mod prelude {
     pub use crate::{
+        analysis::morpheme::Morpheme,
+        analysis::{Mode, Tokenize},
         error::SudachiError,
         error::SudachiResult,
-        morpheme::Morpheme,
-        tokenizer::{Mode, Tokenize},
     };
 }
diff --git a/sudachi/src/plugin/oov/mecab_oov/mod.rs b/sudachi/src/plugin/oov/mecab_oov/mod.rs
index 33fa1c8..8863acc 100644
--- a/sudachi/src/plugin/oov/mecab_oov/mod.rs
+++ b/sudachi/src/plugin/oov/mecab_oov/mod.rs
@@ -21,13 +21,13 @@ use std::fs;
 use std::io::{BufRead, BufReader};
 use std::path::PathBuf;
 
+use crate::analysis::node::Node;
 use crate::config::Config;
 use crate::dic::category_type::CategoryType;
 use crate::dic::character_category::Error as CharacterCategoryError;
 use crate::dic::grammar::Grammar;
 use crate::dic::lexicon::word_infos::WordInfo;
 use crate::input_text::Utf8InputText;
-use crate::lattice::node::Node;
 use crate::plugin::oov::OovProviderPlugin;
 use crate::prelude::*;
 
diff --git a/sudachi/src/plugin/oov/mod.rs b/sudachi/src/plugin/oov/mod.rs
index 8675f7d..5765893 100644
--- a/sudachi/src/plugin/oov/mod.rs
+++ b/sudachi/src/plugin/oov/mod.rs
@@ -16,10 +16,10 @@
 
 use serde_json::Value;
 
+use crate::analysis::node::Node;
 use crate::config::Config;
 use crate::dic::grammar::Grammar;
 use crate::input_text::Utf8InputText;
-use crate::lattice::node::Node;
 use crate::plugin::oov::mecab_oov::MeCabOovPlugin;
 use crate::plugin::oov::simple_oov::SimpleOovPlugin;
 use crate::plugin::PluginCategory;
diff --git a/sudachi/src/plugin/oov/simple_oov/mod.rs b/sudachi/src/plugin/oov/simple_oov/mod.rs
index f010dbd..2a213df 100644
--- a/sudachi/src/plugin/oov/simple_oov/mod.rs
+++ b/sudachi/src/plugin/oov/simple_oov/mod.rs
@@ -17,11 +17,11 @@
 use serde::Deserialize;
 use serde_json::Value;
 
+use crate::analysis::node::Node;
 use crate::config::Config;
 use crate::dic::grammar::Grammar;
 use crate::dic::lexicon::word_infos::WordInfo;
 use crate::input_text::Utf8InputText;
-use crate::lattice::node::Node;
 use crate::plugin::oov::OovProviderPlugin;
 use crate::prelude::*;
 
diff --git a/sudachi/src/plugin/path_rewrite/join_katakana_oov/mod.rs b/sudachi/src/plugin/path_rewrite/join_katakana_oov/mod.rs
index e91dcfe..00ab1a0 100644
--- a/sudachi/src/plugin/path_rewrite/join_katakana_oov/mod.rs
+++ b/sudachi/src/plugin/path_rewrite/join_katakana_oov/mod.rs
@@ -17,11 +17,11 @@
 use serde::Deserialize;
 use serde_json::Value;
 
+use crate::analysis::{lattice::Lattice, node::Node};
 use crate::config::Config;
 use crate::dic::category_type::CategoryType;
 use crate::dic::grammar::Grammar;
 use crate::input_text::Utf8InputText;
-use crate::lattice::{node::Node, Lattice};
 use crate::plugin::path_rewrite::PathRewritePlugin;
 use crate::prelude::*;
 
diff --git a/sudachi/src/plugin/path_rewrite/join_numeric/mod.rs b/sudachi/src/plugin/path_rewrite/join_numeric/mod.rs
index 84ae546..e14c01a 100644
--- a/sudachi/src/plugin/path_rewrite/join_numeric/mod.rs
+++ b/sudachi/src/plugin/path_rewrite/join_numeric/mod.rs
@@ -18,11 +18,11 @@ use serde::Deserialize;
 use serde_json::Value;
 
 use self::numeric_parser::NumericParser;
+use crate::analysis::{lattice::Lattice, node::Node};
 use crate::config::Config;
 use crate::dic::category_type::CategoryType;
 use crate::dic::grammar::Grammar;
 use crate::input_text::Utf8InputText;
-use crate::lattice::{node::Node, Lattice};
 use crate::plugin::path_rewrite::PathRewritePlugin;
 use crate::prelude::*;
 
diff --git a/sudachi/src/plugin/path_rewrite/mod.rs b/sudachi/src/plugin/path_rewrite/mod.rs
index fb7d092..0f6eb09 100644
--- a/sudachi/src/plugin/path_rewrite/mod.rs
+++ b/sudachi/src/plugin/path_rewrite/mod.rs
@@ -19,11 +19,11 @@ pub mod join_numeric;
 
 use serde_json::Value;
 
+use crate::analysis::{lattice::Lattice, node::Node};
 use crate::config::Config;
 use crate::dic::grammar::Grammar;
 use crate::dic::lexicon::word_infos::WordInfo;
 use crate::input_text::Utf8InputText;
-use crate::lattice::{node::Node, Lattice};
 use crate::plugin::path_rewrite::join_katakana_oov::JoinKatakanaOovPlugin;
 use crate::plugin::path_rewrite::join_numeric::JoinNumericPlugin;
 use crate::plugin::PluginCategory;


Test patch:
diff --git a/sudachi/tests/common/mod.rs b/sudachi/tests/common/mod.rs
index dead55b..cf550f3 100644
--- a/sudachi/tests/common/mod.rs
+++ b/sudachi/tests/common/mod.rs
@@ -18,11 +18,11 @@ use std::path::{Path, PathBuf};
 
 extern crate sudachi;
 use self::sudachi::dic::dictionary::JapaneseDictionary;
-use self::sudachi::stateless_tokeniser::StatelessTokenizer;
 use std::fs;
 use std::fs::File;
 use std::io::Read;
 use std::sync::Arc;
+use sudachi::analysis::stateless_tokenizer::StatelessTokenizer;
 use sudachi::config::Config;
 use sudachi::dic::{grammar::Grammar, header::Header, lexicon::Lexicon};
 use sudachi::prelude::*;
diff --git a/sudachi/tests/stateless_tokenizer.rs b/sudachi/tests/stateless_tokenizer.rs
index 6f7b813..7c1441a 100644
--- a/sudachi/tests/stateless_tokenizer.rs
+++ b/sudachi/tests/stateless_tokenizer.rs
@@ -18,7 +18,7 @@
 extern crate lazy_static;
 extern crate sudachi;
 
-use sudachi::tokenizer::Mode;
+use sudachi::prelude::Mode;
 
 mod common;
 use crate::common::TestTokenizer;