Repository: WorksApplications/SudachiPy

Issue:
Slow building of user dictionary with surface form split info
It seems take 30+ seconds per entry to build user dictionary, if its words have split info with word info (構成語情報). 

```
$ sudachipy ubuild user.csv -o user.dic
reading the source file...1 words
writing the POS table...2 bytes
writing the connection matrix...4 bytes
building the trie...done
writing the trie...1028 bytes
writing the word-ID table...9 bytes
writing the word parameters...10 bytes
writing the word_infos...70 bytes
writing word_info offsets...4 bytes

real	0m38.654s
user	0m38.499s
sys	0m0.139s
```

user.csv:

```csv
舞台藝術,5146,5146,8000,舞台藝術,名詞,普通名詞,一般,*,*,*,ブタイゲイジュツ,舞台芸術,*,C,"舞台,名詞,普通名詞,一般,*,*,*,ブタイ/藝術,名詞,普通名詞,一般,*,*,*,ゲイジュツ","舞台,名詞,普通名詞,一般,*,*,*,ブタイ/藝術,名詞,普通名詞,一般,*,*,*,ゲイジュツ","舞台,名詞,普通名詞,一般,*,*,*,ブタイ/藝術,名詞,普通名詞,一般,*,*,*,ゲイジュツ",*
```

If I have two lines then it takes 1m.

※ No problem with **word id** split info

```csv
舞台藝術,5146,5146,8000,舞台藝術,名詞,普通名詞,一般,*,*,*,ブタイゲイジュツ,舞台芸術,*,C,647312/659236,647312/659236,647312/659236,*

```
```
$ sudachipy ubuild user.csv -o user.dic
real	0m0.925s
user	0m0.776s
sys	0m0.159s
```

----

```
$ sudachipy --version
sudachipy 0.5.2
```


Comments:

Patch:
diff --git a/sudachipy/dictionarylib/dictionarybuilder.py b/sudachipy/dictionarylib/dictionarybuilder.py
index 5f3512a..bde5871 100644
--- a/sudachipy/dictionarylib/dictionarybuilder.py
+++ b/sudachipy/dictionarylib/dictionarybuilder.py
@@ -334,7 +334,7 @@ class DictionaryBuilder(object):
 
     @staticmethod
     def __is_id(text):
-        return re.match(r'U?\d+', text)
+        return re.match(r'U?\d+$', text)
 
     def parse_id(self, text):
         if text.startswith('U'):
diff --git a/sudachipy/dictionarylib/doublearraylexicon.py b/sudachipy/dictionarylib/doublearraylexicon.py
index 6e1a7ce..fb77ee8 100644
--- a/sudachipy/dictionarylib/doublearraylexicon.py
+++ b/sudachipy/dictionarylib/doublearraylexicon.py
@@ -78,14 +78,22 @@ class DoubleArrayLexicon(Lexicon):
         return self.word_params.size
 
     def get_word_id(self, headword: str, pos_id: int, reading_form: str) -> int:
+        for wid, _ in self.lookup(headword.encode('utf-8'), 0):
+            if self._compare_word_id(wid, headword, pos_id, reading_form):
+                return wid
+
         for wid in range(self.word_infos.size()):
-            info = self.word_infos.get_word_info(wid)
-            if info.surface == headword \
-                    and info.pos_id == pos_id \
-                    and info.reading_form == reading_form:
+            if self._compare_word_id(wid, headword, pos_id, reading_form):
                 return wid
+
         return -1
 
+    def _compare_word_id(self, wid: int, headword: str, pos_id: int, reading_form: str) -> bool:
+        info = self.word_infos.get_word_info(wid)
+        return info.surface == headword \
+            and info.pos_id == pos_id \
+            and info.reading_form == reading_form
+
     def get_dictionary_id(self, word_id: int) -> int:
         return 0
 


Test patch:
diff --git a/tests/dictionarylib/test_dictionarybuilder.py b/tests/dictionarylib/test_dictionarybuilder.py
index f1ead7a..a07a647 100644
--- a/tests/dictionarylib/test_dictionarybuilder.py
+++ b/tests/dictionarylib/test_dictionarybuilder.py
@@ -39,6 +39,8 @@ class TestDictionaryBuilder(TestCase):
             wf.write("東京都,0,0,0,東京都,名詞,固有名詞,地名,一般,*,*,ヒガシキョウト,東京都,*,B,\"東,名詞,普通名詞,一般,*,*,*,ヒガシ/2\",*,1/2,1/2\n")
             wf.write("東,-1,-1,0,東,名詞,普通名詞,一般,*,*,*,ヒガシ,ひがし,*,A,*,*,*,*\n")
             wf.write("京都,0,0,0,京都,名詞,固有名詞,地名,一般,*,*,キョウト,京都,*,A,*,*,*,*\n")
+            wf.write("1,-1,-1,0,1,名詞,数詞,*,*,*,*,イチ,1,*,A,*,*,*,*\n")
+            wf.write("東1,0,0,0,東1,名詞,普通名詞,一般,*,*,*,ヒガシイチ,東1,*,C,\"東,名詞,普通名詞,一般,*,*,*,ヒガシ/1,名詞,数詞,*,*,*,*,イチ\",\"東,名詞,普通名詞,一般,*,*,*,ヒガシ/U3\",*")
         self.logger = getLogger()
         self.logger.disabled = True
 
@@ -218,13 +220,13 @@ class TestDictionaryBuilder(TestCase):
         self.assertEqual('test', header.description)
 
         # grammar
-        self.assertEqual(2, grammar.get_part_of_speech_size())
+        self.assertEqual(3, grammar.get_part_of_speech_size())
         self.assertEqual(["名詞", "固有名詞", "地名", "一般", "*", "*"], grammar.get_part_of_speech_string(0))
         self.assertEqual(["名詞", "普通名詞", "一般", "*", "*", "*"], grammar.get_part_of_speech_string(1))
         self.assertEqual(200, grammar.get_connect_cost(0, 0))
 
         # lexicon
-        self.assertEqual(3, lexicon.size())
+        self.assertEqual(5, lexicon.size())
         self.assertEqual(0, lexicon.get_cost(0))
         wi = lexicon.get_word_info(0)
         self.assertEqual('東京都', wi.surface)