Repository: nltk/nltk

Issue:
More Python3.9 deprecations
Running pytest with the new Python 3.9.4 shows a number of deprecation warnings about future failures to anticipate:

============================= test session starts ==============================
platform linux -- Python 3.9.4, pytest-6.2.0, py-1.10.0, pluggy-0.13.1

[....]
=============================== warnings summary ===============================
chunk.doctest::chunk.doctest
  <doctest chunk.doctest[11]>:1: DeprecationWarning: invalid escape sequence \.

classify.doctest::classify.doctest
classify.doctest::classify.doctest
classify.doctest::classify.doctest
classify.doctest::classify.doctest
classify.doctest::classify.doctest
nltk/util.py:64: DeprecationWarning: `formatargspec` is deprecated since Python 3.5. Use `signature` and the `Signature` object directly
    argspec = inspect.formatargspec(args, varargs, varkw, defaults)

corpus.doctest::corpus.doctest
  <doctest corpus.doctest[232]>:2: DeprecationWarning: invalid escape sequence \.

corpus.doctest::corpus.doctest
  <doctest corpus.doctest[272]>:1: DeprecationWarning: invalid escape sequence \.

data.doctest::data.doctest
  <doctest data.doctest[63]>:1: DeprecationWarning: 
    Function BufferedGzipFile() has been deprecated.  Use gzip.GzipFile
    instead as it also uses a buffer.

data.doctest::data.doctest
  <doctest data.doctest[67]>:1: DeprecationWarning: 
    Function BufferedGzipFile() has been deprecated.  Use gzip.GzipFile
    instead as it also uses a buffer.

gensim.doctest::gensim.doctest
  <doctest gensim.doctest[7]>:1: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).

gensim.doctest::gensim.doctest
  /usr/lib64/python3.9/site-packages/gensim/models/keyedvectors.py:877: FutureWarning: arrays to stack must be passed as a "sequence" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.
    vectors = vstack(self.word_vec(word, use_norm=True) for word in used_words).astype(REAL)

gluesemantics.doctest::gluesemantics.doctest
  <doctest gluesemantics.doctest[55]>:1: DeprecationWarning: invalid escape sequence \P

gluesemantics.doctest::gluesemantics.doctest
  <doctest gluesemantics.doctest[65]>:1: DeprecationWarning: invalid escape sequence \P

gluesemantics.doctest::gluesemantics.doctest
  <doctest gluesemantics.doctest[71]>:1: DeprecationWarning: invalid escape sequence \P

probability.doctest: 410 warnings
nltk/tag/hmm.py:396: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.
  Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
    B = -np.ones((T, N), np.int)

probability.doctest::probability.doctest
nltk/probability.py:1456: UserWarning: SimpleGoodTuring did not find a proper best fit line for smoothing probabilities of occurrences. The probability estimates are likely to be unreliable.
    warnings.warn(

relextract.doctest::relextract.doctest
  <doctest relextract.doctest[15]>:1: DeprecationWarning: invalid escape sequence \s

toolbox.doctest::toolbox.doctest
  /usr/lib64/python3.9/codecs.py:905: DeprecationWarning: 'U' mode is deprecated
    file = builtins.open(filename, mode, buffering)

toolbox.doctest::toolbox.doctest
  <doctest toolbox.doctest[90]>:1: DeprecationWarning: invalid escape sequence \|

[....]

Discussion:
stevenbird: Thanks @dannysepler, @tomaarsen, @iliakur 

Patch:
diff --git a/nltk/util.py b/nltk/util.py
index c25a8fb33..383342a4e 100644
--- a/nltk/util.py
+++ b/nltk/util.py
@@ -37,32 +37,38 @@ from nltk.internals import raise_unorderable_types, slice_bounds
 ######################################################################
 
 
-def usage(obj, selfname="self"):
+def usage(obj):
     str(obj)  # In case it's lazy, this will load it.
 
     if not isinstance(obj, type):
         obj = obj.__class__
 
-    print("%s supports the following operations:" % obj.__name__)
+    print(f"{obj.__name__} supports the following operations:")
     for (name, method) in sorted(pydoc.allmethods(obj).items()):
         if name.startswith("_"):
             continue
         if getattr(method, "__deprecated__", False):
             continue
 
-        getargspec = inspect.getfullargspec
-        args, varargs, varkw, defaults = getargspec(method)[:4]
-        if (
-            args
-            and args[0] == "self"
-            and (defaults is None or len(args) > len(defaults))
-        ):
-            args = args[1:]
-            name = f"{selfname}.{name}"
-        argspec = inspect.formatargspec(args, varargs, varkw, defaults)
+        try:
+            sig = str(inspect.signature(method))
+        except ValueError as e:
+            # builtins sometimes don't support introspection
+            if "builtin" in str(e):
+                continue
+            else:
+                raise
+
+        args = sig.lstrip("(").rstrip(")").split(", ")
+        meth = inspect.getattr_static(obj, name)
+        if isinstance(meth, (classmethod, staticmethod)):
+            name = f"cls.{name}"
+        elif args and args[0] == "self":
+            name = f"self.{name}"
+            args.pop(0)
         print(
             textwrap.fill(
-                f"{name}{argspec}",
+                f"{name}({', '.join(args)})",
                 initial_indent="  - ",
                 subsequent_indent=" " * (len(name) + 5),
             )


Test patch:
diff --git a/nltk/test/unit/test_util.py b/nltk/test/unit/test_util.py
index 365452574..109a96b31 100644
--- a/nltk/test/unit/test_util.py
+++ b/nltk/test/unit/test_util.py
@@ -1,81 +1,134 @@
-"""
-Unit tests for nltk.util.
-"""
-
-import unittest
-
-from nltk.util import everygrams
-
-
-class TestEverygrams(unittest.TestCase):
-    def setUp(self):
-        """Form test data for tests."""
-        self.test_data = iter("a b c".split())
-
-    def test_everygrams_without_padding(self):
-        expected_output = [
-            ("a",),
-            ("a", "b"),
-            ("a", "b", "c"),
-            ("b",),
-            ("b", "c"),
-            ("c",),
-        ]
-        output = everygrams(self.test_data)
-        self.assertCountEqual(output, expected_output)
-
-    def test_everygrams_max_len(self):
-        expected_output = [
-            ("a",),
-            ("a", "b"),
-            ("b",),
-            ("b", "c"),
-            ("c",),
-        ]
-        output = everygrams(self.test_data, max_len=2)
-        self.assertCountEqual(output, expected_output)
-
-    def test_everygrams_min_len(self):
-        expected_output = [
-            ("a", "b"),
-            ("b", "c"),
-            ("a", "b", "c"),
-        ]
-        output = everygrams(self.test_data, min_len=2)
-        self.assertCountEqual(output, expected_output)
-
-    def test_everygrams_pad_right(self):
-        expected_output = [
-            ("a",),
-            ("a", "b"),
-            ("a", "b", "c"),
-            ("b",),
-            ("b", "c"),
-            ("b", "c", None),
-            ("c",),
-            ("c", None),
-            ("c", None, None),
-            (None,),
-            (None, None),
-            (None,),
-        ]
-        output = everygrams(self.test_data, max_len=3, pad_right=True)
-        self.assertCountEqual(output, expected_output)
-
-    def test_everygrams_pad_left(self):
-        expected_output = [
-            (None,),
-            (None, None),
-            (None, None, "a"),
-            (None,),
-            (None, "a"),
-            (None, "a", "b"),
-            ("a",),
-            ("a", "b"),
-            ("a", "b", "c"),
-            ("b",),
-            ("b", "c"),
-            ("c",),
-        ]
-        output = everygrams(self.test_data, max_len=3, pad_left=True)
-        self.assertCountEqual(output, expected_output)
+import pytest
+
+from nltk.util import everygrams, usage
+
+
+def test_usage_with_self(capsys):
+    class MyClass:
+        def kwargs(self, a=1):
+            ...
+
+        def no_args(self):
+            ...
+
+        def pos_args(self, a, b):
+            ...
+
+        def pos_args_and_kwargs(self, a, b, c=1):
+            ...
+
+    usage(MyClass)
+
+    captured = capsys.readouterr()
+    assert captured.out == (
+        "MyClass supports the following operations:\n"
+        "  - self.kwargs(a=1)\n"
+        "  - self.no_args()\n"
+        "  - self.pos_args(a, b)\n"
+        "  - self.pos_args_and_kwargs(a, b, c=1)\n"
+    )
+
+
+def test_usage_with_cls(capsys):
+    class MyClass:
+        @classmethod
+        def clsmethod(cls):
+            ...
+
+        @classmethod
+        def clsmethod_with_args(cls, a, b, c=1):
+            ...
+
+    usage(MyClass)
+
+    captured = capsys.readouterr()
+    assert captured.out == (
+        "MyClass supports the following operations:\n"
+        "  - cls.clsmethod()\n"
+        "  - cls.clsmethod_with_args(a, b, c=1)\n"
+    )
+
+
+def test_usage_on_builtin():
+    # just check the func passes, since
+    # builtins change each python version
+    usage(dict)
+
+
+@pytest.fixture
+def everygram_input():
+    """Form test data for tests."""
+    return iter(["a", "b", "c"])
+
+
+def test_everygrams_without_padding(everygram_input):
+    expected_output = [
+        ("a",),
+        ("a", "b"),
+        ("a", "b", "c"),
+        ("b",),
+        ("b", "c"),
+        ("c",),
+    ]
+    output = list(everygrams(everygram_input))
+    assert output == expected_output
+
+
+def test_everygrams_max_len(everygram_input):
+    expected_output = [
+        ("a",),
+        ("a", "b"),
+        ("b",),
+        ("b", "c"),
+        ("c",),
+    ]
+    output = list(everygrams(everygram_input, max_len=2))
+    assert output == expected_output
+
+
+def test_everygrams_min_len(everygram_input):
+    expected_output = [
+        ("a", "b"),
+        ("a", "b", "c"),
+        ("b", "c"),
+    ]
+    output = list(everygrams(everygram_input, min_len=2))
+    assert output == expected_output
+
+
+def test_everygrams_pad_right(everygram_input):
+    expected_output = [
+        ("a",),
+        ("a", "b"),
+        ("a", "b", "c"),
+        ("b",),
+        ("b", "c"),
+        ("b", "c", None),
+        ("c",),
+        ("c", None),
+        ("c", None, None),
+        (None,),
+        (None, None),
+        (None,),
+    ]
+    output = list(everygrams(everygram_input, max_len=3, pad_right=True))
+    assert output == expected_output
+
+
+def test_everygrams_pad_left(everygram_input):
+    expected_output = [
+        (None,),
+        (None, None),
+        (None, None, "a"),
+        (None,),
+        (None, "a"),
+        (None, "a", "b"),
+        ("a",),
+        ("a", "b"),
+        ("a", "b", "c"),
+        ("b",),
+        ("b", "c"),
+        ("c",),
+    ]
+    output = list(everygrams(everygram_input, max_len=3, pad_left=True))
+    assert output == expected_output