Repository: mocobeta/janome

Issue:
IndexError: list index out of rangeと表示されます。
卒業制作でクローラーを制作しながらPythonを勉強しているのですが、このようなエラーが出て困っています。何が原因なのかさっぱりわからないのでお力を貸して頂けないでしょうか？
よろしくお願いします。

文字列として渡す値に問題があるような気がするのですが、リスト配列からはみ出ているというのがよくわかりません。

**エラー内容**
```
Traceback (most recent call last):
  File "/vagrant/pysearch-master/manage.py", line 15, in <module>
    crawl_web('https://news.google.com/news/headlines?hl=ja&ned=jp', 3)
  File "/vagrant/pysearch-master/web_crawler/crawler.py", line 137, in crawl_web
    add_page_to_index(page_url, html)
  File "/vagrant/pysearch-master/web_crawler/crawler.py", line 121, in add_page_to_index
    for word in _split_to_word(line): #janomeで日本語形態解析して単語をwordに代入
  File "/vagrant/pysearch-master/web_crawler/crawler.py", line 45, in _split_to_word
    return [token.surface for token in t.tokenize(text)]
  File "/home/vagrant/.virtualenvs/dev/local/lib/python3.4/site-packages/janome/tokenizer.py", line 193, in tokenize
    return list(self.__tokenize_stream(text, wakati))
  File "/home/vagrant/.virtualenvs/dev/local/lib/python3.4/site-packages/janome/tokenizer.py", line 200, in __tokenize_stream
    tokens, pos = self.__tokenize_partial(text[processed:], wakati)
  File "/home/vagrant/.virtualenvs/dev/local/lib/python3.4/site-packages/janome/tokenizer.py", line 254, in __tokenize_partial
    lattice.end()
  File "/home/vagrant/.virtualenvs/dev/local/lib/python3.4/site-packages/janome/lattice.py", line 154, in end
    self.add(eos)
  File "/home/vagrant/.virtualenvs/dev/local/lib/python3.4/site-packages/janome/lattice.py", line 140, in add
    node.index = len(self.snodes[self.p])
IndexError: list index out of range
```

**htmlからbody以下にあるタグの中にあるテキストを解析に渡す**
```
def add_page_to_index(url, html):
    body_soup = BeautifulSoup(html, "html.parser").find('body')
    #htmlないの属性タグとその中身をchild_tagに入れていってる<body>以下にある全てのタグ<a>やら<th>やらを持ってくる。
    #先ずはbodyより下のhtml全部持ってきて次にその下のdivを持ってきてul持ってきてどんどん掘り下げる感じ
    #if body_soup.findChildren() is not None:
    for child_tag in body_soup.findChildren():
        #beautifulsoupの機能でタグの名前だけ取り出してる。スクリプトだけは避ける。それ以降の処理がスキップされてループに戻る
        if child_tag.name == 'script': #child_tag.nameタグの名前を取り出す
            continue
        #.textはそのタグの中身を表示する。<a>link</a>だったらlinkだけとりだす。
        child_text = child_tag.text
        for line in child_text.split('\n'): #文字列から改行を取り除いて分ける。
            line = line.rstrip().lstrip() #上のコードだけだと両端の空白が消せないからここで削除している。実際には削除はできないので取り除いたのを返している
            for word in _split_to_word(line): #janomeで日本語形態解析して単語をwordに代入
                add_to_index(word, url)
```
**add_page_to_indexで文字列で受け取ったテキストを解析して単語にして返す**
```
def _split_to_word(text):
    """Japanese morphological analysis with janome.
    Splitting text and creating words list.
    """
    t = Tokenizer()
    #token.surfaceで日本語の文字だけ取り出せる。例えば車は高いだったら"車" "は" "高い" だけ取り出せる。
    return [token.surface for token in t.tokenize(text)]
```



Comments:

Patch:
diff --git a/janome/lattice.py b/janome/lattice.py
index 27a2046..0a9388d 100644
--- a/janome/lattice.py
+++ b/janome/lattice.py
@@ -100,7 +100,7 @@ class EOS(object):
 class Lattice:
     def __init__(self, size, dic):
         self.snodes = [[BOS()]] + [[] for i in range(0, size + 1)]
-        self.enodes = [[], [BOS()]] + [[] for i in range(0, size + 2)]
+        self.enodes = [[], [BOS()]] + [[] for i in range(0, size + 1)]
         self.conn_costs = [[]]
         self.p = 1
         self.dic = dic
diff --git a/janome/tokenizer.py b/janome/tokenizer.py
index 5606c90..ccc476b 100644
--- a/janome/tokenizer.py
+++ b/janome/tokenizer.py
@@ -141,7 +141,7 @@ class Tokenizer:
     A Tokenizer tokenizes Japanese texts with system and optional user defined dictionary.
     It is strongly recommended to re-use a Tokenizer object because object initialization cost is high. 
     """
-    MAX_CHUNK_SIZE = 1000
+    MAX_CHUNK_SIZE = 1024
     CHUNK_SIZE = 500
 
     def __init__(self, udic='', udic_enc='utf8', udic_type='ipadic', max_unknown_length=1024, wakati=False, mmap=False):
@@ -212,7 +212,7 @@ class Tokenizer:
         lattice = Lattice(chunk_size, self.sys_dic)
         pos = 0
         while not self.__should_split(text, pos):
-            encoded_partial_text = text[pos:pos+min(50, len(text)-pos)].encode('utf-8')
+            encoded_partial_text = text[pos:pos+min(50, chunk_size-pos)].encode('utf-8')
             # user dictionary
             if self.user_dic:
                 entries = self.user_dic.lookup(encoded_partial_text)
@@ -238,7 +238,7 @@ class Tokenizer:
                     assert length >= 0
                     # buffer for unknown word
                     buf = text[pos]
-                    for p in range(pos + 1, min(len(text), pos + length + 1)):
+                    for p in range(pos + 1, min(chunk_size, pos + length + 1)):
                         _cates =  self.sys_dic.get_char_categories(text[p])
                         if cate in _cates or any(cate in _compat_cates for _compat_cates in _cates.values()):
                             buf += text[p]


Test patch:
diff --git a/tests/test_lattice.py b/tests/test_lattice.py
index cf4b890..14d6a66 100644
--- a/tests/test_lattice.py
+++ b/tests/test_lattice.py
@@ -33,7 +33,7 @@ class TestLattice(unittest.TestCase):
         lattice = Lattice(5, SYS_DIC)
         self.assertEqual(7, len(lattice.snodes))
         self.assertTrue(isinstance(lattice.snodes[0][0], BOS))
-        self.assertEqual(9, len(lattice.enodes))
+        self.assertEqual(8, len(lattice.enodes))
         self.assertTrue(isinstance(lattice.enodes[1][0], BOS))
 
     def test_add_forward_end(self):
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
index cc249d2..1928cf5 100644
--- a/tests/test_tokenizer.py
+++ b/tests/test_tokenizer.py
@@ -173,6 +173,13 @@ class TestTokenizer(unittest.TestCase):
                 text = unicode(text)
             tokens = Tokenizer().tokenize(text)
 
+    def test_tokenize_large_text3(self):
+        with open('tests/text_large_nonjp.txt', encoding='utf-8') as f:
+            text = f.read()
+            if not PY3:
+                text = unicode(text)
+            tokens = Tokenizer().tokenize(text)
+
     def test_tokenize_large_text_stream(self):
         with open('tests/text_lemon.txt', encoding='utf-8') as f:
             text = f.read()
@@ -187,6 +194,13 @@ class TestTokenizer(unittest.TestCase):
                 text = unicode(text)
             tokens = list(Tokenizer().tokenize(text, stream = True))
 
+    def test_tokenize_large_text_stream3(self):
+        with open('tests/text_large_nonjp.txt', encoding='utf-8') as f:
+            text = f.read()
+            if not PY3:
+                text = unicode(text)
+            tokens = list(Tokenizer().tokenize(text, stream = True))
+
     def test_tokenize_wakati(self):
         text = u'すもももももももものうち'
         tokens = Tokenizer(wakati = True).tokenize(text, wakati = True)
diff --git a/tests/text_large_nonjp.txt b/tests/text_large_nonjp.txt
new file mode 100644
index 0000000..fa61527
--- /dev/null
+++ b/tests/text_large_nonjp.txt
@@ -0,0 +1,1 @@
+longlonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglongtext