Repository: arushadev/piraye (language: python)

Issue:
Tokenize double quotes
https://stackoverflow.com/a/32197336/3343474


Discussion / hints:

Fix patch:
diff --git a/setup.py b/setup.py
index d1a95f8..2490772 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@
         "Normalization",
     ],
     url="https://github.com/arushadev/piraye",
-    version="0.1.0",
+    version="0.1.1",
     package_dir={"piraye": "src"},
     packages=["piraye"],
     package_data={"piraye": ["data/*/*.json"]},
diff --git a/src/__init__.py b/src/__init__.py
index fe655de..e2db95f 100644
--- a/src/__init__.py
+++ b/src/__init__.py
@@ -1,5 +1,6 @@
 """This module includes Normalizer and NormalizerBuilder"""
 from .normalizer import Normalizer
 from .normalizer_builder import NormalizerBuilder
+from .nltk_tokenizer import NltkTokenizer
 
-__all__ = ["Normalizer", "NormalizerBuilder"]
+__all__ = ["Normalizer", "NormalizerBuilder", "NltkTokenizer"]
diff --git a/src/tokenizer.py b/src/nltk_tokenizer.py
similarity index 88%
rename from src/tokenizer.py
rename to src/nltk_tokenizer.py
index 0069af8..ba8fdeb 100644
--- a/src/tokenizer.py
+++ b/src/nltk_tokenizer.py
@@ -90,8 +90,11 @@ def __get_original_tokens(text: str, text2: str, tokens_en: List[str]) -> List[s
         text2_counter = 0
         tokens = []
         for token_en in tokens_en:
-            token_index = text2.index(token_en, text2_counter)
-            curr_text = text[token_index:token_index + len(token_en)]
-            tokens.append(curr_text)
-            text2_counter = token_index + len(token_en)
+            try:
+                token_index = text2.index(token_en, text2_counter)
+                curr_text = text[token_index:token_index + len(token_en)]
+                tokens.append(curr_text)
+                text2_counter = token_index + len(token_en)
+            except ValueError:
+                tokens.append(token_en)
         return tokens
diff --git a/src/normalizer.py b/src/normalizer.py
index 9cb3a4f..cfb437f 100644
--- a/src/normalizer.py
+++ b/src/normalizer.py
@@ -5,7 +5,7 @@
 
 from .char_config import CharConfig
 from .mappings import MappingDict
-from .tokenizer import NltkTokenizer, Tokenizer
+from .nltk_tokenizer import NltkTokenizer, Tokenizer
 
 
 # pylint: disable=too-few-public-methods


Test patch:
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
new file mode 100644
index 0000000..d649b4d
--- /dev/null
+++ b/tests/test_tokenizer.py
@@ -0,0 +1,21 @@
+# testing Fibonacci number function
+# pylint: skip-file
+
+from ..src import NltkTokenizer
+
+
+def test_object():
+    tokenizer = NltkTokenizer()
+    assert tokenizer is not None
+
+
+def test_sample():
+    text = "برای تست شماره ۲.۱ نوشته شده است"
+    tokenizer = NltkTokenizer()
+    assert len(tokenizer.word_tokenize(text)) == 7
+
+
+def test_double_quotes():
+    text = "\"\"تست\""
+    tokenizer = NltkTokenizer()
+    assert len(tokenizer.word_tokenize(text)) == 4