Repository: nltk/nltk

Issue:
SpanishStemmer raises IndexError on 'algue'
"algue" is the only word it happens out of many tens of thousands I went through.

To reproduce:

``` python
from nltk.stem.snowball import SpanishStemmer
stemmer = SpanishStemmer()
stemmer.stem('algue')
```

``` python
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/local/lib/python2.7/site-packages/nltk/stem/snowball.py", line 3555, in stem
    if word[-2:] == u"gu" and rv[-1] == u"u":
IndexError: string index out of range
```

Happens both with nltk 2.0.4 and 3.0.0



Discussion:
To be IndexError-safe, that should be rv[-1:] not rv[-1]. You can submit a
PR...

On 7 October 2014 22:44, Miquel Torres notifications@github.com wrote:

>  "algue" is the only word it happens out of many tens of thousands I went
> through.
> 
> To reproduce:
> 
> from nltk.stem.snowball import SpanishStemmerstemmer = SpanishStemmer()stemmer.stem('algue')
> 
>  Traceback (most recent call last):
>   File "<stdin>", line 1, in <module>
>   File "/local/lib/python2.7/site-packages/nltk/stem/snowball.py", line 3555, in stem
>     if word[-2:] == u"gu" and rv[-1] == u"u":IndexError: string index out of range
> 
>  Happens both with nltk 2.0.4 and 3.0.0
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/nltk/nltk/issues/759.


Resolving patch:
diff --git a/nltk/stem/snowball.py b/nltk/stem/snowball.py
--- a/nltk/stem/snowball.py
+++ b/nltk/stem/snowball.py
@@ -26,6 +26,7 @@
 
 from nltk.stem.api import StemmerI
 
+
 class SnowballStemmer(StemmerI):
 
     """
@@ -189,7 +190,6 @@ def _r1_scandinavian(self, word, vowels):
         return r1
 
 
-
 class _StandardStemmer(_LanguageSpecificStemmer):
 
     """
@@ -3551,7 +3551,7 @@ def stem(self, word):
                     word = word[:-len(suffix)]
                     rv = rv[:-len(suffix)]
 
-                    if word[-2:] == "gu" and rv[-1] == "u":
+                    if word[-2:] == "gu" and rv[-1:] == "u":
                         word = word[:-1]
                 else:
                     word = word[:-len(suffix)]
@@ -3561,11 +3561,9 @@ def stem(self, word):
                     .replace("\xED", "i").replace("\xF3", "o")
                     .replace("\xFA", "u"))
 
-
         return word
 
 
-
 class SwedishStemmer(_ScandinavianStemmer):
 
     """
@@ -3644,11 +3642,9 @@ def stem(self, word):
                     word = word[:-1]
                 break
 
-
         return word
 
 
-
 def demo():
     """
     This function provides a demonstration of the Snowball stemmers.
@@ -3720,8 +3716,6 @@ def demo():
         print("\n")
 
 
-
 if __name__ == "__main__":
     import doctest
     doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
-


Test patch:
diff --git a/nltk/test/unit/test_stem.py b/nltk/test/unit/test_stem.py
--- a/nltk/test/unit/test_stem.py
+++ b/nltk/test/unit/test_stem.py
@@ -3,6 +3,7 @@
 import unittest
 from nltk.stem.snowball import SnowballStemmer
 
+
 class SnowballTest(unittest.TestCase):
 
     def test_russian(self):
@@ -22,6 +23,14 @@ def test_german(self):
         assert stemmer_german.stem("keinen") == 'kein'
         assert stemmer_german2.stem("keinen") == 'keinen'
 
+    def test_spanish(self):
+        stemmer = SnowballStemmer('spanish')
+
+        assert stemmer.stem("Visionado") == 'vision'
+
+        # The word 'algue' was raising an IndexError
+        assert stemmer.stem("algue") == 'algu'
+
     def test_short_strings_bug(self):
         stemmer = SnowballStemmer('english')
         assert stemmer.stem("y's") == 'y'