Repository: nltk/nltk

Issue:
Sonority sequencing syllable tokenizer performs significantly slower on numbers than on words
The sonority sequencing syllable tokenizer (`nltk.SyllableTokenizer`) performs significantly slower on numbers than on words. It seems that the time complexity for words is O(n), which is okay, but O(n^2) for numbers, which is not so good.

```
>>> timeit.timeit('t.tokenize("99")', setup='import nltk; t = nltk.SyllableTokenizer()', number = 100)
0.03364099999998871
>>> timeit.timeit('t.tokenize("thisisanextremelylongword")', setup='import nltk; t = nltk.SyllableTokenizer()', number = 100)
0.002708099999949809
>>> timeit.timeit('t.tokenize("99")', setup='import nltk; t = nltk.SyllableTokenizer()', number = 1000)
2.5833234000000402
>>> timeit.timeit('t.tokenize("thisisanextremelylongword")', setup='import nltk; t = nltk.SyllableTokenizer()', number = 1000)
0.023796200000106182
>>> timeit.timeit('t.tokenize("99")', setup='import nltk; t = nltk.SyllableTokenizer()', number = 10000)
264.43897390000006
>>> timeit.timeit('t.tokenize("thisisanextremelylongword")', setup='import nltk; t = nltk.SyllableTokenizer()', number = 10000)
0.24109669999984362
```

OS: Windows 10 x64
Python: 3.8.10 x64
NLTK: 3.7


Discussion:

Resolving patch:
diff --git a/nltk/tokenize/sonority_sequencing.py b/nltk/tokenize/sonority_sequencing.py
--- a/nltk/tokenize/sonority_sequencing.py
+++ b/nltk/tokenize/sonority_sequencing.py
@@ -98,14 +98,15 @@ def assign_values(self, token):
             try:
                 syllables_values.append((c, self.phoneme_map[c]))
             except KeyError:
-                if c not in punctuation:
+                if c not in "0123456789" and c not in punctuation:
                     warnings.warn(
                         "Character not defined in sonority_hierarchy,"
                         " assigning as vowel: '{}'".format(c)
                     )
                     syllables_values.append((c, max(self.phoneme_map.values())))
-                    self.vowels += c
-                else:  # If it's a punctuation, assign -1.
+                    if c not in self.vowels:
+                        self.vowels += c
+                else:  # If it's a punctuation or numbers, assign -1.
                     syllables_values.append((c, -1))
         return syllables_values
 
@@ -122,11 +123,12 @@ def validate_syllables(self, syllable_list):
         """
         valid_syllables = []
         front = ""
+        vowel_pattern = re.compile("|".join(self.vowels))
         for i, syllable in enumerate(syllable_list):
             if syllable in punctuation:
                 valid_syllables.append(syllable)
                 continue
-            if not re.search("|".join(self.vowels), syllable):
+            if not vowel_pattern.search(syllable):
                 if len(valid_syllables) == 0:
                     front += syllable
                 else:


Test patch:
diff --git a/nltk/test/unit/test_tokenize.py b/nltk/test/unit/test_tokenize.py
--- a/nltk/test/unit/test_tokenize.py
+++ b/nltk/test/unit/test_tokenize.py
@@ -270,6 +270,15 @@ def test_sonority_sequencing_syllable_tokenizer(self):
         tokens = tokenizer.tokenize("justification")
         assert tokens == ["jus", "ti", "fi", "ca", "tion"]
 
+    def test_syllable_tokenizer_numbers(self):
+        """
+        Test SyllableTokenizer tokenizer.
+        """
+        tokenizer = SyllableTokenizer()
+        text = "9" * 10000
+        tokens = tokenizer.tokenize(text)
+        assert tokens == [text]
+
     def test_legality_principle_syllable_tokenizer(self):
         """
         Test LegalitySyllableTokenizer tokenizer.