Repository: nltk/nltk

Issue:
NLTK 3.2.1: Deprecated Warning with Python 3.5.2
Hi,
I'm learning Python.
For my first program in Python, I downloaded Chatterbot 0.4.6 that contained nltk module. 

This program doesn't work because of nltk/decorators.py generates a Deprecated Warning (inspect.getargspec). I installed last version of NLTK with the process explained in the website www.nltk.org but I have the same problem.
This is the result when I try to run the program Chatterbot 0.4.6:

Warning (from warnings module):
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/nltk/decorators.py", line 59
    regargs, varargs, varkwargs, defaults = inspect.getargspec(func)
DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead

Can you help me?

Can't import nltk in Python 3.6 - AttributeError: module 'inspect' has no attribute 'getargspec'
I'm running this under Mac OS X in a virtualenv.

$ ipython
Python **3.6.0a0** (default, Sep 24 2015, 09:10:51) 
Type "copyright", "credits" or "license" for more information.

IPython 5.1.0 -- An enhanced Interactive Python.
?         -> Introduction and overview of IPython's features.
%quickref -> Quick reference.
help      -> Python's own help system.
object?   -> Details about 'object', use 'object??' for extra details.

In [1]: import nltk

AttributeError                            Traceback (most recent call last)
<ipython-input-1-b06499430ee0> in <module>()
----> 1 import nltk

/Users/test/venv/lib/python3.6/site-packages/nltk/__init__.py in <module>()
    113 
    114 from nltk.collocations import *
--> 115 from nltk.decorators import decorator, memoize
    116 from nltk.featstruct import *
    117 from nltk.grammar import *

/Users/test/venv/lib/python3.6/site-packages/nltk/decorators.py in <module>()
    181         return default
    182 
--> 183 @decorator
    184 def memoize(func, *args):
    185     dic = getattr_(func, "memoize_dic", dict)

/Users/test/venv/lib/python3.6/site-packages/nltk/decorators.py in decorator(caller)
    170         dec_func = eval(src, dict(_func_=func, _call_=caller))
    171         return update_wrapper(dec_func, func, infodict)
--> 172     return update_wrapper(_decorator, caller)
    173 
    174 def getattr_(obj, name, default_thunk):

/Users/test/venv/lib/python3.6/site-packages/nltk/decorators.py in update_wrapper(wrapper, model, infodict)
     81 # akin to functools.update_wrapper
     82 def update_wrapper(wrapper, model, infodict=None):
---> 83     infodict = infodict or getinfo(model)
     84     wrapper.__name__ = infodict['name']
     85     wrapper.__doc__ = infodict['doc']

/Users/test/venv/lib/python3.6/site-packages/nltk/decorators.py in getinfo(func)
     57     """
     58     assert inspect.ismethod(func) or inspect.isfunction(func)
---> 59     regargs, varargs, varkwargs, defaults = inspect.getargspec(func)
     60     argnames = list(regargs)
     61     if varargs:

AttributeError: module 'inspect' has no attribute 'getargspec'




Discussion:
ChatterBot solved the problem (see gunthercox/ChatterBot#292).
We still need to check it.

Works fine with python 3.5!  :)
Indeed, Python3.6 is another can of worms yet to be handled in `nltk`. Python 3.4 -> 3.5 was quite a headache. Any brave takers to port `nltk` for Py3.6+ ?

A good start would be to read the PEPs on https://docs.python.org/3.6/whatsnew/3.6.html

Resolving patch:
diff --git a/nltk/decorators.py b/nltk/decorators.py
--- a/nltk/decorators.py
+++ b/nltk/decorators.py
@@ -56,7 +56,11 @@ def getinfo(func):
     'self, x, y, *args, **kw'
     """
     assert inspect.ismethod(func) or inspect.isfunction(func)
-    regargs, varargs, varkwargs, defaults = inspect.getargspec(func)
+    if sys.version_info[0] >= 3:
+        argspec = inspect.getfullargspec(func)
+    else:
+        argspec = inspect.getargspec(func)
+    regargs, varargs, varkwargs, defaults = argspec[:4]
     argnames = list(regargs)
     if varargs:
         argnames.append(varargs)
diff --git a/nltk/sem/evaluate.py b/nltk/sem/evaluate.py
--- a/nltk/sem/evaluate.py
+++ b/nltk/sem/evaluate.py
@@ -19,6 +19,7 @@
 import inspect
 import textwrap
 import re
+import sys
 
 from nltk.decorators import decorator # this used in code that is commented out
 from nltk.compat import string_types, python_2_unicode_compatible
@@ -36,7 +37,10 @@ class Error(Exception): pass
 class Undefined(Error):  pass
 
 def trace(f, *args, **kw):
-    argspec = inspect.getargspec(f)
+    if sys.version_info[0] >= 3:
+        argspec = inspect.getfullargspec(f)
+    else:
+        argspec = inspect.getargspec(f)
     d = dict(zip(argspec[0], args))
     if d.pop('trace', None):
         print()
diff --git a/nltk/util.py b/nltk/util.py
--- a/nltk/util.py
+++ b/nltk/util.py
@@ -6,6 +6,8 @@
 # For license information, see LICENSE.TXT
 from __future__ import print_function
 
+import sys
+import inspect
 import locale
 import re
 import types
@@ -32,7 +34,6 @@
 ######################################################################
 
 def usage(obj, selfname='self'):
-    import inspect
     str(obj) # In case it's lazy, this will load it.
 
     if not isinstance(obj, class_types):
@@ -43,7 +44,11 @@ def usage(obj, selfname='self'):
         if name.startswith('_'): continue
         if getattr(method, '__deprecated__', False): continue
 
-        args, varargs, varkw, defaults = inspect.getargspec(method)
+        if sys.version_info[0] >= 3:
+            getargspec = inspect.getfullargspec
+        else:
+            getargspec = inspect.getargspec
+        args, varargs, varkw, defaults = getargspec(method)[:4]
         if (args and args[0]=='self' and
             (defaults is None or len(args)>len(defaults))):
             args = args[1:]
@@ -379,18 +384,18 @@ def flatten(*args):
 # Ngram iteration
 ##########################################################################
 
-def pad_sequence(sequence, n, pad_left=False, pad_right=False, 
+def pad_sequence(sequence, n, pad_left=False, pad_right=False,
                  left_pad_symbol=None, right_pad_symbol=None):
     """
     Returns a padded sequence of items before ngram extraction.
-    
+
         >>> list(pad_sequence([1,2,3,4,5], 2, pad_left=True, pad_right=True, left_pad_symbol='
', right_pad_symbol='
'))
         ['
', 1, 2, 3, 4, 5, '
']
         >>> list(pad_sequence([1,2,3,4,5], 2, pad_left=True, left_pad_symbol='
'))
         ['
', 1, 2, 3, 4, 5]
         >>> list(pad_sequence([1,2,3,4,5], 2, pad_right=True, right_pad_symbol='
'))
         [1, 2, 3, 4, 5, '
']
-    
+
     :param sequence: the source data to be padded
     :type sequence: sequence or iter
     :param n: the degree of the ngrams
@@ -414,7 +419,7 @@ def pad_sequence(sequence, n, pad_left=False, pad_right=False,
 
 # add a flag to pad the sequence so we get peripheral ngrams?
 
-def ngrams(sequence, n, pad_left=False, pad_right=False, 
+def ngrams(sequence, n, pad_left=False, pad_right=False,
            left_pad_symbol=None, right_pad_symbol=None):
     """
     Return the ngrams generated from a sequence of items, as an iterator.
@@ -453,7 +458,7 @@ def ngrams(sequence, n, pad_left=False, pad_right=False,
     """
     sequence = pad_sequence(sequence, n, pad_left, pad_right,
                             left_pad_symbol, right_pad_symbol)
-        
+
     history = []
     while n > 1:
         history.append(next(sequence))
@@ -504,13 +509,13 @@ def trigrams(sequence, **kwargs):
 def everygrams(sequence, min_len=1, max_len=-1, **kwargs):
     """
     Returns all possible ngrams generated from a sequence of items, as an iterator.
-    
+
         >>> sent = 'a b c'.split()
         >>> list(everygrams(sent))
         [('a',), ('b',), ('c',), ('a', 'b'), ('b', 'c'), ('a', 'b', 'c')]
         >>> list(everygrams(sent, max_len=2))
         [('a',), ('b',), ('c',), ('a', 'b'), ('b', 'c')]
-        
+
     :param sequence: the source data to be converted into trigrams
     :type sequence: sequence or iter
     :param min_len: minimum length of the ngrams, aka. n-gram order/degree of ngram
@@ -519,7 +524,7 @@ def everygrams(sequence, min_len=1, max_len=-1, **kwargs):
     :type  max_len: int
     :rtype: iter(tuple)
     """
-    
+
     if max_len == -1:
         max_len = len(sequence)
     for n in range(min_len, max_len+1):
@@ -531,13 +536,13 @@ def skipgrams(sequence, n, k, **kwargs):
     Returns all possible skipgrams generated from a sequence of items, as an iterator.
     Skipgrams are ngrams that allows tokens to be skipped.
     Refer to http://homepages.inf.ed.ac.uk/ballison/pdf/lrec_skipgrams.pdf
-    
+
         >>> sent = "Insurgents killed in ongoing fighting".split()
         >>> list(skipgrams(sent, 2, 2))
         [('Insurgents', 'killed'), ('Insurgents', 'in'), ('Insurgents', 'ongoing'), ('killed', 'in'), ('killed', 'ongoing'), ('killed', 'fighting'), ('in', 'ongoing'), ('in', 'fighting'), ('ongoing', 'fighting')]
         >>> list(skipgrams(sent, 3, 2))
         [('Insurgents', 'killed', 'in'), ('Insurgents', 'killed', 'ongoing'), ('Insurgents', 'killed', 'fighting'), ('Insurgents', 'in', 'ongoing'), ('Insurgents', 'in', 'fighting'), ('Insurgents', 'ongoing', 'fighting'), ('killed', 'in', 'ongoing'), ('killed', 'in', 'fighting'), ('killed', 'ongoing', 'fighting'), ('in', 'ongoing', 'fighting')]
-    
+
     :param sequence: the source data to be converted into trigrams
     :type sequence: sequence or iter
     :param n: the degree of the ngrams
@@ -546,13 +551,13 @@ def skipgrams(sequence, n, k, **kwargs):
     :type  k: int
     :rtype: iter(tuple)
     """
-    
+
     # Pads the sequence as desired by **kwargs.
     if 'pad_left' in kwargs or 'pad_right' in kwargs:
         sequence = pad_sequence(sequence, n, **kwargs)
-    
+
     # Note when iterating through the ngrams, the pad_right here is not
-    # the **kwargs padding, it's for the algorithm to detect the SENTINEL 
+    # the **kwargs padding, it's for the algorithm to detect the SENTINEL
     # object on the right pad to stop inner loop.
     SENTINEL = object()
     for ngram in ngrams(sequence, n + k, pad_right=True, right_pad_symbol=SENTINEL):
@@ -686,9 +691,9 @@ def elementtree_indent(elem, level=0):
     """
     Recursive function to indent an ElementTree._ElementInterface
     used for pretty printing. Run indent on elem and then output
-    in the normal way. 
-    
-    :param elem: element to be indented. will be modified. 
+    in the normal way.
+
+    :param elem: element to be indented. will be modified.
     :type elem: ElementTree._ElementInterface
     :param level: level of indentation for this element
     :type level: nonnegative integer
@@ -715,17 +720,17 @@ def elementtree_indent(elem, level=0):
 def choose(n, k):
     """
     This function is a fast way to calculate binomial coefficients, commonly
-    known as nCk, i.e. the number of combinations of n things taken k at a time. 
+    known as nCk, i.e. the number of combinations of n things taken k at a time.
     (https://en.wikipedia.org/wiki/Binomial_coefficient).
-    
-    This is the *scipy.special.comb()* with long integer computation but this 
+
+    This is the *scipy.special.comb()* with long integer computation but this
     approximation is faster, see https://github.com/nltk/nltk/issues/1181
-    
+
         >>> choose(4, 2)
         6
         >>> choose(6, 2)
         15
-    
+
     :param n: The number of things.
     :type n: int
     :param r: The number of times a thing is taken.


Test patch: