Repository: WorksApplications/sudachi.rs (language: python)

Issue:
Support unicode combining  symbols
e.g. in emojis like 👨‍👨‍👦‍👦 which are encoded as 7 codepoints: 4 people and 3 combining marks

It should be forbidden to begin OOVs on them and on the next symbol after them

## Variants to handle

Notation: A and C are some symbols, B is a combining mark.

1. ABC -> D (e.g. emoji)
2. AB -> C (most of unicode combining marks https://en.wikipedia.org/wiki/Combining_character)
3. ABC -> AC https://en.wikipedia.org/wiki/Zero-width_non-joiner

### Handling AB -> C

B should be marked as NOBOOV, B should be of ANY character category

### Handling ABC -> D

B and C should be marked as NOBOOV, B should be of ANY character category

### Handling ABC -> AC

B and C should be marked as NOBOOV, B should be of ANY character category (?)


Discussion / hints:

Fix patch:
diff --git a/python/py_src/sudachipy/resources/char.def b/python/py_src/sudachipy/resources/char.def
index b9728b79..90985b4b 100644
--- a/python/py_src/sudachipy/resources/char.def
+++ b/python/py_src/sudachipy/resources/char.def
@@ -133,7 +133,6 @@ CYRILLIC        1 1 0
 0x2000..0x206F  SYMBOL # General Punctuation
 0x2070..0x209F  NUMERIC # Superscripts and Subscripts
 0x20A0..0x20CF  SYMBOL # Currency Symbols
-0x20D0..0x20FF  SYMBOL # Combining Diaritical Marks for Symbols
 0x2100..0x214F  SYMBOL # Letterlike Symbols
 0x2150..0x218F  NUMERIC # Number forms
 0x2100..0x214B  SYMBOL # Letterlike Symbols
@@ -158,10 +157,23 @@ CYRILLIC        1 1 0
 0xFE30..0xFE4F  SYMBOL # CJK Compatibility Forms
 0xFE50..0xFE6B  SYMBOL # Small Form Variants
 
-# added 2006/3/13 
+# added 2006/3/13
 0x3007 SYMBOL KANJINUMERIC
 
 # added 2018/11/30
 0x309b..0x309c HIRAGANA KATAKANA # voiced/semi-voiced sound marks
 
+# Unicode combining symbols 2021/12/03
+# from https://en.wikipedia.org/wiki/Combining_character
+0x0300..0x036F ALL NOOOVBOW # Combining Diacritical Marks
+0x1AB0..0x1AFF ALL NOOOVBOW # Combining Diacritical Marks Extended
+0x1DC0..0x1DFF ALL NOOOVBOW # Combining Diacritical Marks Supplement
+0x20D0..0x20FF ALL NOOOVBOW # Combining Diacritical Marks for Symbols
+0xFE20..0xFE2F ALL NOOOVBOW # Combining Half Marks
+0xFE00..0xFE0F ALL NOOOVBOW # https://codepoints.net/variation_selectors
+0x1F3FB..0x1F3FE ALL NOOOVBOW # emoji skin tone modifiers https://codepoints.net/U+1F3FF
+
+# Combination marks
+0x200C..0x200D ALL NOOOVBOW2 # Zero Width Non-Joiner/Joiner
+
 # END OF TABLE
diff --git a/resources/char.def b/resources/char.def
index b9728b79..83e789a0 100644
--- a/resources/char.def
+++ b/resources/char.def
@@ -133,7 +133,6 @@ CYRILLIC        1 1 0
 0x2000..0x206F  SYMBOL # General Punctuation
 0x2070..0x209F  NUMERIC # Superscripts and Subscripts
 0x20A0..0x20CF  SYMBOL # Currency Symbols
-0x20D0..0x20FF  SYMBOL # Combining Diaritical Marks for Symbols
 0x2100..0x214F  SYMBOL # Letterlike Symbols
 0x2150..0x218F  NUMERIC # Number forms
 0x2100..0x214B  SYMBOL # Letterlike Symbols
@@ -164,4 +163,17 @@ CYRILLIC        1 1 0
 # added 2018/11/30
 0x309b..0x309c HIRAGANA KATAKANA # voiced/semi-voiced sound marks
 
+# Unicode combining symbols 2021/12/03
+# from https://en.wikipedia.org/wiki/Combining_character
+0x0300..0x036F ALL NOOOVBOW # Combining Diacritical Marks
+0x1AB0..0x1AFF ALL NOOOVBOW # Combining Diacritical Marks Extended
+0x1DC0..0x1DFF ALL NOOOVBOW # Combining Diacritical Marks Supplement
+0x20D0..0x20FF ALL NOOOVBOW # Combining Diacritical Marks for Symbols
+0xFE20..0xFE2F ALL NOOOVBOW # Combining Half Marks
+0xFE00..0xFE0F ALL NOOOVBOW # https://codepoints.net/variation_selectors
+0x1F3FB..0x1F3FE ALL NOOOVBOW # emoji skin tone modifiers https://codepoints.net/U+1F3FF
+
+# Combination marks
+0x200C..0x200D ALL NOOOVBOW2 # Zero Width Non-Joiner/Joiner
+
 # END OF TABLE
diff --git a/sudachi/src/analysis/stateful_tokenizer.rs b/sudachi/src/analysis/stateful_tokenizer.rs
index b00b5045..f4aadd52 100644
--- a/sudachi/src/analysis/stateful_tokenizer.rs
+++ b/sudachi/src/analysis/stateful_tokenizer.rs
@@ -220,16 +220,13 @@ impl<D: DictionaryAccess> StatefulTokenizer<D> {
         lattice.reset(input.current_chars().len());
 
         for (ch_off, &byte_off) in input.curr_byte_offsets().iter().enumerate() {
-            if !input.can_bow(byte_off) {
-                continue;
-            }
-
             if !lattice.has_previous_node(ch_off) {
                 continue;
             }
 
             let mut has_word = false;
             for e in dict.lexicon().lookup(input_bytes, byte_off) {
+                // do we really need input.can_bow condition?
                 if (e.end < input_bytes.len()) && !input.can_bow(e.end) {
                     continue;
                 }
@@ -248,7 +245,10 @@ impl<D: DictionaryAccess> StatefulTokenizer<D> {
             }
 
             // OOV
-            if !input.cat_at_char(ch_off).contains(CategoryType::NOOOVBOW) {
+            if !input
+                .cat_at_char(ch_off)
+                .intersects(CategoryType::NOOOVBOW | CategoryType::NOOOVBOW2)
+            {
                 for oov_provider in dict.oov_provider_plugins() {
                     oov_provider.get_oov(&input, ch_off, has_word, oovs)?;
                 }
diff --git a/sudachi/src/config.rs b/sudachi/src/config.rs
index 6ec273ab..f3bf78d6 100644
--- a/sudachi/src/config.rs
+++ b/sudachi/src/config.rs
@@ -17,7 +17,7 @@
 use std::env::current_exe;
 use std::fs::File;
 use std::io::BufReader;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use lazy_static::lazy_static;
 use serde::Deserialize;
@@ -48,7 +48,7 @@ pub enum ConfigError {
 }
 
 /// Setting data loaded from config file
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct Config {
     pub resource_dir: PathBuf,
     pub system_dict: Option<PathBuf>,
@@ -66,7 +66,7 @@ pub struct Config {
 /// For plugins, refer to each plugin.
 #[allow(non_snake_case)]
 #[derive(Deserialize, Debug)]
-pub struct RawConfig {
+pub struct ConfigBuilder {
     resourcePath: Option<PathBuf>,
     systemDict: Option<PathBuf>,
     userDict: Option<Vec<PathBuf>>,
@@ -78,60 +78,111 @@ pub struct RawConfig {
     pathRewritePlugin: Option<Vec<Value>>,
 }
 
-impl Config {
-    pub fn new(
-        config_file: Option<PathBuf>,
-        resource_dir: Option<PathBuf>,
-        dictionary_path: Option<PathBuf>,
-    ) -> Result<Self, ConfigError> {
+impl ConfigBuilder {
+    pub fn from_file(config_file: &Path) -> Result<Self, ConfigError> {
+        let file = File::open(config_file)?;
+        let reader = BufReader::new(file);
+        serde_json::from_reader(reader).map_err(|e| e.into())
+    }
+
+    pub fn from_bytes(data: &[u8]) -> Result<Self, ConfigError> {
+        serde_json::from_slice(data).map_err(|e| e.into())
+    }
+
+    pub fn empty() -> Self {
+        serde_json::from_slice(b"{}").unwrap()
+    }
+
+    pub fn system_dict(mut self, dict: impl Into<PathBuf>) -> Self {
+        self.systemDict = Some(dict.into());
+        self
+    }
+
+    pub fn user_dict(mut self, dict: impl Into<PathBuf>) -> Self {
+        let dicts = match self.userDict.as_mut() {
+            None => {
+                self.userDict = Some(Default::default());
+                self.userDict.as_mut().unwrap()
+            }
+            Some(dicts) => dicts,
+        };
+        dicts.push(dict.into());
+        self
+    }
+
+    pub fn resource_path(mut self, path: impl Into<PathBuf>) -> Self {
+        self.resourcePath = Some(path.into());
+        self
+    }
+
+    pub fn build(self) -> Config {
         let src_root_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
         let default_resource_dir_path = src_root_path.join("..").join(DEFAULT_RESOURCE_DIR);
 
-        // prioritize arg (cli option) > default
-        let config_file = match config_file {
-            Some(v) => v,
-            None => default_resource_dir_path.join(DEFAULT_SETTING_FILE),
-        };
-        let file = File::open(config_file)?;
-        let reader = BufReader::new(file);
-        let raw_config: RawConfig = serde_json::from_reader(reader)?;
+        let resource_dir = self.resourcePath.unwrap_or(default_resource_dir_path);
 
-        // prioritize arg (cli option) > config file > default
-        let resource_dir = resource_dir
-            .or_else(|| raw_config.resourcePath.clone())
-            .unwrap_or_else(|| default_resource_dir_path);
+        let system_dict = self
+            .systemDict
+            .clone()
+            .map(|p| Config::join_if_relative(&resource_dir, p));
 
-        // prioritize arg (cli option) > config file
-        let system_dict = dictionary_path.or_else(|| {
-            raw_config
-                .systemDict
-                .clone()
-                .map(|p| Config::join_if_relative(&resource_dir, p))
-        });
-        let user_dicts = raw_config
+        let user_dicts = self
             .userDict
             .unwrap_or(Vec::new())
             .into_iter()
             .map(|p| Config::join_if_relative(&resource_dir, p))
             .collect();
+
         let character_definition_file = Config::join_if_relative(
             &resource_dir,
-            raw_config
-                .characterDefinitionFile
+            self.characterDefinitionFile
                 .unwrap_or(PathBuf::from(DEFAULT_CHAR_DEF_FILE)),
         );
 
-        Ok(Config {
+        Config {
             resource_dir,
             system_dict,
             user_dicts,
             character_definition_file,
 
-            connection_cost_plugins: raw_config.connectionCostPlugin.unwrap_or(Vec::new()),
-            input_text_plugins: raw_config.inputTextPlugin.unwrap_or(Vec::new()),
-            oov_provider_plugins: raw_config.oovProviderPlugin.unwrap_or(Vec::new()),
-            path_rewrite_plugins: raw_config.pathRewritePlugin.unwrap_or(Vec::new()),
-        })
+            connection_cost_plugins: self.connectionCostPlugin.unwrap_or(Vec::new()),
+            input_text_plugins: self.inputTextPlugin.unwrap_or(Vec::new()),
+            oov_provider_plugins: self.oovProviderPlugin.unwrap_or(Vec::new()),
+            path_rewrite_plugins: self.pathRewritePlugin.unwrap_or(Vec::new()),
+        }
+    }
+}
+
+impl Config {
+    pub fn new(
+        config_file: Option<PathBuf>,
+        resource_dir: Option<PathBuf>,
+        dictionary_path: Option<PathBuf>,
+    ) -> Result<Self, ConfigError> {
+        let src_root_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        let default_resource_dir_path = src_root_path.join("..").join(DEFAULT_RESOURCE_DIR);
+
+        // prioritize arg (cli option) > default
+        let config_file = match config_file {
+            Some(v) => v,
+            None => default_resource_dir_path.join(DEFAULT_SETTING_FILE),
+        };
+
+        let raw_config = ConfigBuilder::from_file(&config_file)?;
+
+        // prioritize arg (cli option) > config file
+        let raw_config = match resource_dir {
+            None => raw_config,
+            Some(p) => raw_config.resource_path(p),
+        };
+
+        // prioritize arg (cli option) > config file
+        let raw_config = match dictionary_path {
+            None => raw_config,
+            Some(p) => raw_config.system_dict(p),
+        };
+
+        Ok(raw_config.build())
     }
 
     /// Creates a minimal config with the provided resource directory
diff --git a/sudachi/src/dic/category_type.rs b/sudachi/src/dic/category_type.rs
index 4407cda3..189df5f6 100644
--- a/sudachi/src/dic/category_type.rs
+++ b/sudachi/src/dic/category_type.rs
@@ -54,11 +54,13 @@ bitflags! {
         const USER3 = (1 << 13);
         /** User defined category. */
         const USER4 = (1 << 14);
-        /** Characters that cannot be the beginning of word */
-        const NOOOVBOW = (1 << 31);
+        /** This character cannot be the beginning of an OOV word */
+        const NOOOVBOW = (1 << 30);
+        /** This and next characters cannot be the beginning of an OOV word */
+        const NOOOVBOW2 = (1 << 31);
 
-        /** All categories at once except NOOOVBOW */
-        const ALL = !(1 << 31);
+        /** All categories at once except NOOOVBOW/2 */
+        const ALL = 0b00111111_11111111_11111111_11111111;
     }
 }
 
@@ -122,6 +124,7 @@ impl FromStr for CategoryType {
             "USER3" => Ok(Self::USER3),
             "USER4" => Ok(Self::USER4),
             "NOOOVBOW" => Ok(Self::NOOOVBOW),
+            "NOOOVBOW2" => Ok(Self::NOOOVBOW2),
             "ALL" => Ok(Self::ALL),
             _ => Err(SudachiError::InvalidCharacterCategoryType(String::from(s))),
         }
diff --git a/sudachi/src/input_text/buffer/mod.rs b/sudachi/src/input_text/buffer/mod.rs
index 91dfa282..db2996af 100644
--- a/sudachi/src/input_text/buffer/mod.rs
+++ b/sudachi/src/input_text/buffer/mod.rs
@@ -144,6 +144,7 @@ impl InputBuffer {
         let non_starting = CategoryType::ALPHA | CategoryType::GREEK | CategoryType::CYRILLIC;
         let mut prev_cat = CategoryType::empty();
         self.mod_bow.resize(self.modified.len(), false);
+        let mut next_bow = true;
 
         for (chidx, (bidx, ch)) in self.modified.char_indices().enumerate() {
             self.mod_chars.push(ch);
@@ -155,12 +156,26 @@ impl InputBuffer {
             last_offset = bidx;
             last_chidx = chidx;
 
-            // BOW logic: for special cases check if the previous char is compatible
-            self.mod_bow[bidx] = if cat.intersects(non_starting) {
+            let can_bow = if !next_bow {
+                // this char was forbidden by the previous one
+                next_bow = true;
+                false
+            } else if cat.intersects(CategoryType::NOOOVBOW2) {
+                // this rule is stronger than the next one and must come before
+                // this and next are forbidden
+                next_bow = false;
+                false
+            } else if cat.intersects(CategoryType::NOOOVBOW) {
+                // this char is forbidden
+                false
+            } else if cat.intersects(non_starting) {
+                // the previous char is compatible
                 !cat.intersects(prev_cat)
             } else {
                 true
             };
+
+            self.mod_bow[bidx] = can_bow;
             prev_cat = cat;
         }
         // trailing indices for the last codepoint


Test patch:
diff --git a/sudachi/tests/common/mod.rs b/sudachi/tests/common/mod.rs
index b2fdf1ef..d69287d4 100644
--- a/sudachi/tests/common/mod.rs
+++ b/sudachi/tests/common/mod.rs
@@ -25,7 +25,7 @@ use std::sync::Arc;
 
 use sudachi::analysis::stateful_tokenizer::StatefulTokenizer;
 use sudachi::analysis::stateless_tokenizer::StatelessTokenizer;
-use sudachi::config::Config;
+use sudachi::config::{Config, ConfigBuilder};
 use sudachi::dic::dictionary::JapaneseDictionary;
 use sudachi::dic::{grammar::Grammar, header::Header, lexicon::Lexicon, DictionaryLoader};
 use sudachi::prelude::*;
@@ -45,6 +45,8 @@ pub fn dictionary_bytes_from_path<P: AsRef<Path>>(dictionary_path: P) -> Sudachi
     Ok(dictionary_bytes)
 }
 
+pub const LEX_CSV: &[u8] = include_bytes!("../resources/lex.csv");
+
 lazy_static! {
     pub static ref TEST_CONFIG: Config = {
         let test_config_path = "tests/resources/sudachi.json";
@@ -117,6 +119,7 @@ pub struct TestTokenizerBuilder<'a> {
     pub user: Vec<&'a [u8]>,
     pub mode: Mode,
     pub debug: bool,
+    pub config: Option<&'a [u8]>,
 }
 
 #[allow(unused)]
@@ -136,6 +139,11 @@ impl<'a> TestTokenizerBuilder<'a> {
         self
     }
 
+    pub fn config(mut self, data: &'static [u8]) -> Self {
+        self.config = Some(data);
+        self
+    }
+
     pub fn build(self) -> TestStatefulTokenizer {
         let mut sys = DictBuilder::new_system();
         sys.read_conn(
@@ -167,7 +175,12 @@ impl<'a> TestTokenizerBuilder<'a> {
             }
         }
 
-        let dic = JapaneseDictionary::from_cfg_storage(&TEST_CONFIG, data).unwrap();
+        let config = match self.config {
+            None => TEST_CONFIG.clone(),
+            Some(data) => ConfigBuilder::from_bytes(data).unwrap().build(),
+        };
+
+        let dic = JapaneseDictionary::from_cfg_storage(&config, data).unwrap();
         let rcdic = Rc::new(dic);
 
         TestStatefulTokenizer {
@@ -199,6 +212,7 @@ impl TestStatefulTokenizer {
             conn: None,
             mode: Mode::C,
             debug: false,
+            config: None,
         }
     }
 
diff --git a/sudachi/tests/resources/char.def b/sudachi/tests/resources/char.def
index 078fcea5..95182375 100644
--- a/sudachi/tests/resources/char.def
+++ b/sudachi/tests/resources/char.def
@@ -1,3 +1,6 @@
+DEFAULT         0 1 0  # DEFAULT is a mandatory category!
+ALPHA           1 1 0
+
 0x0030..0x0039 NUMERIC  #0-9
 0x0041..0x005A ALPHA    #A-Z
 0x0061..0x007A ALPHA    #a-z
@@ -18,3 +21,16 @@
 0xFF10..0xFF19 NUMERIC
 0xFF21..0xFF3A ALPHA
 0xFF41..0xFF5A ALPHA
+
+# Unicode combining symbols 2021/12/03
+# from https://en.wikipedia.org/wiki/Combining_character
+0x0300..0x036F ALL NOOOVBOW # Combining Diacritical Marks
+0x1AB0..0x1AFF ALL NOOOVBOW # Combining Diacritical Marks Extended
+0x1DC0..0x1DFF ALL NOOOVBOW # Combining Diacritical Marks Supplement
+0x20D0..0x20FF ALL NOOOVBOW # Combining Diacritical Marks for Symbols
+0xFE20..0xFE2F ALL NOOOVBOW # Combining Half Marks
+0xFE00..0xFE0F ALL NOOOVBOW # https://codepoints.net/variation_selectors
+0x1F3FB..0x1F3FE ALL NOOOVBOW # emoji skin tone modifiers https://codepoints.net/U+1F3FF
+
+# Combination marks
+0x200C..0x200D ALL NOOOVBOW2 # Zero Width Non-Joiner/Joiner
\ No newline at end of file
diff --git a/sudachi/tests/resources/lex.csv b/sudachi/tests/resources/lex.csv
new file mode 100644
index 00000000..5b8ff325
--- /dev/null
+++ b/sudachi/tests/resources/lex.csv
@@ -0,0 +1,46 @@
+た,1,1,8729,た,助動詞,*,*,*,助動詞-タ,終止形-一般,タ,た,*,A,*,*,*,*
+に,2,2,11406,に,助詞,接続助詞,*,*,*,*,ニ,に,*,A,*,*,*,*
+に,3,3,4481,に,助詞,格助詞,*,*,*,*,ニ,に,*,A,*,*,*,*
+京都,6,6,5293,京都,名詞,固有名詞,地名,一般,*,*,キョウト,京都,*,A,*,*,*,1/5
+東,7,7,4675,東,名詞,普通名詞,一般,*,*,*,ヒガシ,東,*,A,*,*,*,*
+東京,6,6,2816,東京,名詞,固有名詞,地名,一般,*,*,トウキョウ,東京,*,A,*,*,*,*
+東京都,6,8,5320,東京都,名詞,固有名詞,地名,一般,*,*,トウキョウト,東京都,*,B,5/9,*,5/9,*
+行く,4,4,5105,行く,動詞,非自立可能,*,*,五段-カ行,終止形-一般,イク,行く,*,A,*,*,*,*
+行っ,5,5,5122,行っ,動詞,非自立可能,*,*,五段-カ行,連用形-促音便,イッ,行く,7,A,*,*,*,*
+都,8,8,2914,都,名詞,普通名詞,一般,*,*,*,ト,都,*,A,*,*,*,*
+アイ,7,7,4675,アイ,名詞,普通名詞,一般,*,*,*,アイ,アイ,*,A,*,*,*,*
+アイウ,7,7,4675,アイウ,名詞,普通名詞,一般,*,*,*,アイウ,アイウ,*,A,*,*,*,*
+アイアイウ,6,6,32766,アイウ,名詞,固有名詞,地名,一般,*,*,アイアイウ,アイアイウ,*,A,*,*,*,*
+0,9,9,2478,0,名詞,数詞,*,*,*,*,ゼロ,0,*,A,*,*,*,*
+1,9,9,2478,1,名詞,数詞,*,*,*,*,イチ,1,*,A,*,*,*,*
+2,9,9,2478,2,名詞,数詞,*,*,*,*,ニ,2,*,A,*,*,*,*
+3,9,9,2478,3,名詞,数詞,*,*,*,*,サン,3,*,A,*,*,*,*
+4,9,9,2478,4,名詞,数詞,*,*,*,*,ヨン,4,*,A,*,*,*,*
+5,9,9,2478,5,名詞,数詞,*,*,*,*,ゴ,5,*,A,*,*,*,*
+6,9,9,2478,6,名詞,数詞,*,*,*,*,ロク,6,*,A,*,*,*,*
+7,9,9,2478,7,名詞,数詞,*,*,*,*,ナナ,7,*,A,*,*,*,*
+8,9,9,2478,8,名詞,数詞,*,*,*,*,ハチ,8,*,A,*,*,*,*
+9,9,9,2478,9,名詞,数詞,*,*,*,*,キュウ,9,*,A,*,*,*,*
+〇,9,9,2478,〇,名詞,数詞,*,*,*,*,ゼロ,〇,*,A,*,*,*,*
+一,9,9,2478,一,名詞,数詞,*,*,*,*,イチ,一,*,A,*,*,*,*
+二,9,9,2478,二,名詞,数詞,*,*,*,*,ニ,二,*,A,*,*,*,*
+三,9,9,2478,三,名詞,数詞,*,*,*,*,サン,三,*,A,*,*,*,*
+四,9,9,2478,四,名詞,数詞,*,*,*,*,ヨン,四,*,A,*,*,*,*
+五,9,9,2478,五,名詞,数詞,*,*,*,*,ゴ,五,*,A,*,*,*,*
+六,9,9,2478,六,名詞,数詞,*,*,*,*,ロク,六,*,A,*,*,*,*
+七,9,9,2478,七,名詞,数詞,*,*,*,*,ナナ,七,*,A,*,*,*,*
+八,9,9,2478,八,名詞,数詞,*,*,*,*,ハチ,八,*,A,*,*,*,*
+九,9,9,2478,九,名詞,数詞,*,*,*,*,キュウ,九,*,A,*,*,*,*
+六三四,6,6,0,六三四,名詞,固有名詞,地名,一般,*,*,ムサシ,六三四,*,A,*,*,*,*
+いく,4,4,5105,いく,動詞,非自立可能,*,*,五段-カ行,終止形-一般,イク,行く,*,A,*,*,*,*
+いっ,5,5,5122,いっ,動詞,非自立可能,*,*,五段-カ行,連用形-促音便,イッ,行く,34,A,*,*,*,*
+012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,9,9,2478,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,名詞,数詞,*,*,*,*,ゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウ,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,*,A,*,*,*,*
+特a,8,8,2914,特a,名詞,普通名詞,一般,*,*,*,トクエー,特a,*,A,*,*,*,*
+な。な,8,8,2914,な。な,名詞,普通名詞,一般,*,*,*,ナナ,な。な,*,A,*,*,*,*
+ ,8,8,-20000, ,空白,*,*,*,*,*, , ,*,A,*,*,*,*
+X,8,8,-20000,X,補助記号,一般,*,*,*,*,X,X,*,A,*,*,*,*
+X,8,8,-20000,X,感動詞,一般,*,*,*,*,X,X,*,A,*,*,*,*
+X,8,8,-20000,X,名詞,普通名詞,サ変可能,*,*,*,X,X,*,A,*,*,*,*
+X,8,8,-20000,X,名詞,固有名詞,一般,*,*,*,X,X,*,A,*,*,*,*
+X,8,8,-20000,X,名詞,固有名詞,人名,一般,*,*,X,X,*,A,*,*,*,*
+X,8,8,-20000,X,名詞,固有名詞,地名,一般,*,*,X,X,*,A,*,*,*,*
\ No newline at end of file
diff --git a/sudachi/tests/resources/sudachi.oov.json b/sudachi/tests/resources/sudachi.oov.json
new file mode 100644
index 00000000..55126540
--- /dev/null
+++ b/sudachi/tests/resources/sudachi.oov.json
@@ -0,0 +1,27 @@
+{
+    "resourcePath" : "tests/resources/",
+    "systemDict" : "system.dic.test",
+    "userDict" : [ "user.dic.test" ],
+    "characterDefinitionFile" : "char.def",
+    "inputTextPlugin" : [
+        { "class" : "com.worksap.nlp.sudachi.DefaultInputTextPlugin" }
+    ],
+    "oovProviderPlugin" : [
+        { "class" : "com.worksap.nlp.sudachi.MeCabOovPlugin",
+          "charDef" : "char.def",
+          "unkDef" : "unk2.def" },
+        { "class" : "com.worksap.nlp.sudachi.SimpleOovPlugin",
+          "oovPOS" : [ "名詞", "普通名詞", "一般", "*", "*", "*" ],
+          "leftId" : 8,
+          "rightId" : 8,
+          "cost" : 6000 }
+    ],
+    "pathRewritePlugin" : [
+        { "class" : "com.worksap.nlp.sudachi.JoinNumericPlugin",
+          "enableNormalize" : true },
+        { "class" : "com.worksap.nlp.sudachi.JoinKatakanaOovPlugin",
+          "oovPOS" : [ "名詞", "普通名詞", "一般", "*", "*", "*" ],
+          "minLength" : 3
+        }
+    ]
+}
diff --git a/sudachi/tests/resources/unk2.def b/sudachi/tests/resources/unk2.def
new file mode 100644
index 00000000..17f31276
--- /dev/null
+++ b/sudachi/tests/resources/unk2.def
@@ -0,0 +1,2 @@
+DEFAULT,7,7,3857,補助記号,一般,*,*,*,*
+ALPHA,7,7,11633,名詞,普通名詞,一般,*,*,*
\ No newline at end of file
diff --git a/sudachi/tests/stateful_tokenizer.rs b/sudachi/tests/stateful_tokenizer.rs
index 482eb5f1..b8541b61 100644
--- a/sudachi/tests/stateful_tokenizer.rs
+++ b/sudachi/tests/stateful_tokenizer.rs
@@ -21,7 +21,7 @@ use std::ops::Deref;
 use sudachi::prelude::Mode;
 
 mod common;
-use crate::common::TestStatefulTokenizer as TestTokenizer;
+use crate::common::{TestStatefulTokenizer as TestTokenizer, LEX_CSV};
 
 #[test]
 fn empty() {
@@ -143,3 +143,21 @@ fn split_middle() {
     assert_eq!(ms_a.get(1).begin(), 12);
     assert_eq!(ms_a.get(1).end(), 15);
 }
+
+const OOV_CFG: &[u8] = include_bytes!("resources/sudachi.oov.json");
+
+#[test]
+fn istanbul_is_not_splitted() {
+    let mut tok = TestTokenizer::builder(LEX_CSV).config(OOV_CFG).build();
+    let ms = tok.tokenize("İstanbul");
+    assert_eq!(ms.len(), 1);
+}
+
+#[test]
+fn emoji_are_not_splitted() {
+    let mut tok = TestTokenizer::builder(LEX_CSV).config(OOV_CFG).build();
+    assert_eq!(tok.tokenize("⏸").len(), 1);
+    assert_eq!(tok.tokenize("🦹‍♂️").len(), 1);
+    assert_eq!(tok.tokenize("🎅🏾").len(), 1);
+    assert_eq!(tok.tokenize("👳🏽‍♂").len(), 1);
+}