Repository: nltk/nltk

Issue:
ValueError when unpickling ParentedTree with Python 3.7 or higher
While working in Python 3.6, when unpickling a ParentedTree with Python 3.7 or Python 3.8 following ValueError occurs:
`ValueError: Can not insert a subtree that already has a parent.`

Following example script produces the Error:

```python
import pickle
from nltk.tree import ParentedTree


tree = ParentedTree.fromstring('(S (NN x) (NP x) (NN x))')
pickled = pickle.dumps(tree)
tree_2 = pickle.loads(pickled)

print(tree)
print(tree_2)
```

Output of Python 3.6 (working):
```
(S (NN x) (NP x) (NN x))
(S (NN x) (NP x) (NN x))
```

Output in Python 3.7 / 3.8:
```
Traceback (most recent call last):
  File "nltk_pickle_test.py", line 7, in <module>
    tree_2 = pickle.loads(pickled)
  File "pickletest/venv3.8/lib/python3.8/site-packages/nltk/tree.py", line 1192, in extend
    self._setparent(child, len(self))
  File "pickletest/venv3.8/lib/python3.8/site-packages/nltk/tree.py", line 1358, in _setparent
    raise ValueError('Can not insert a subtree that already ' 'has a parent.')
ValueError: Can not insert a subtree that already has a parent.
```

The error also occurs when saving and loading via a file. Tested with nltk 3.4.5.

Discussion:

Patch:
diff --git a/nltk/__init__.py b/nltk/__init__.py
index 1cd72fb45..a96ac22b2 100644
--- a/nltk/__init__.py
+++ b/nltk/__init__.py
@@ -135,7 +135,6 @@ from nltk.featstruct import *
 from nltk.grammar import *
 from nltk.probability import *
 from nltk.text import *
-from nltk.tree import *
 from nltk.util import *
 from nltk.jsontags import *
 
@@ -151,6 +150,7 @@ from nltk.parse import *
 from nltk.tag import *
 from nltk.tokenize import *
 from nltk.translate import *
+from nltk.tree import *
 from nltk.sem import *
 from nltk.stem import *
 
@@ -200,7 +200,7 @@ else:
 from nltk import ccg, chunk, classify, collocations
 from nltk import data, featstruct, grammar, help, inference, metrics
 from nltk import misc, parse, probability, sem, stem, wsd
-from nltk import tag, tbl, text, tokenize, translate, tree, treetransforms, util
+from nltk import tag, tbl, text, tokenize, translate, tree, util
 
 
 # FIXME:  override any accidentally imported demo, see https://github.com/nltk/nltk/issues/2116
diff --git a/nltk/corpus/__init__.py b/nltk/corpus/__init__.py
index e951c6028..d348810a4 100644
--- a/nltk/corpus/__init__.py
+++ b/nltk/corpus/__init__.py
@@ -361,6 +361,9 @@ wordnet = LazyCorpusLoader(
     WordNetCorpusReader,
     LazyCorpusLoader("omw", CorpusReader, r".*/wn-data-.*\.tab", encoding="utf8"),
 )
+wordnet31 = LazyCorpusLoader("wordnet31", WordNetCorpusReader, None)
+# wordnet2021 is scheduled for release in 2021 :)
+# wordnet2021 = LazyCorpusLoader("wordnet2021", WordNetCorpusReader, None)
 wordnet_ic = LazyCorpusLoader("wordnet_ic", WordNetICCorpusReader, r".*\.dat")
 words = LazyCorpusLoader(
     "words", WordListCorpusReader, r"(?!README|\.).*", encoding="ascii"
diff --git a/nltk/corpus/reader/bracket_parse.py b/nltk/corpus/reader/bracket_parse.py
index 92602e4a8..1088842b8 100644
--- a/nltk/corpus/reader/bracket_parse.py
+++ b/nltk/corpus/reader/bracket_parse.py
@@ -53,9 +53,7 @@ class BracketParseCorpusReader(SyntaxCorpusReader):
             for normalizing or converting the POS tags returned by the
             ``tagged_...()`` methods.
         """
-        # FIXME: Why is it inheritting from SyntaxCorpusReader but initializing
-        #       from CorpusReader?
-        CorpusReader.__init__(self, root, fileids, encoding)
+        SyntaxCorpusReader.__init__(self, root, fileids, encoding)
         self._comment_char = comment_char
         self._detect_blocks = detect_blocks
         self._tagset = tagset
diff --git a/nltk/corpus/reader/dependency.py b/nltk/corpus/reader/dependency.py
index 26ceb7067..bc6f10757 100644
--- a/nltk/corpus/reader/dependency.py
+++ b/nltk/corpus/reader/dependency.py
@@ -23,9 +23,7 @@ class DependencyCorpusReader(SyntaxCorpusReader):
         sent_tokenizer=RegexpTokenizer("\n", gaps=True),
         para_block_reader=read_blankline_block,
     ):
-        # FIXME: Why is it inheritting from SyntaxCorpusReader but initializing
-        #       from CorpusReader?
-        CorpusReader.__init__(self, root, fileids, encoding)
+        SyntaxCorpusReader.__init__(self, root, fileids, encoding)
 
     #########################################################
 
diff --git a/nltk/corpus/reader/knbc.py b/nltk/corpus/reader/knbc.py
index b898fc24a..4b1c3bf27 100644
--- a/nltk/corpus/reader/knbc.py
+++ b/nltk/corpus/reader/knbc.py
@@ -58,9 +58,7 @@ class KNBCorpusReader(SyntaxCorpusReader):
         morphs2str is a function to convert morphlist to str for tree representation
         for _parse()
         """
-        # FIXME: Why is it inheritting from SyntaxCorpusReader but initializing
-        #       from CorpusReader?
-        CorpusReader.__init__(self, root, fileids, encoding)
+        SyntaxCorpusReader.__init__(self, root, fileids, encoding)
         self.morphs2str = morphs2str
 
     def _read_block(self, stream):
diff --git a/nltk/corpus/reader/wordnet.py b/nltk/corpus/reader/wordnet.py
index 580ea3a85..3b49d8223 100644
--- a/nltk/corpus/reader/wordnet.py
+++ b/nltk/corpus/reader/wordnet.py
@@ -32,6 +32,7 @@ http://compling.hss.ntu.edu.sg/omw/
 
 import math
 import re
+import warnings
 from collections import defaultdict, deque
 from functools import total_ordering
 from itertools import chain, islice
@@ -1130,6 +1131,11 @@ class WordNetCorpusReader(CorpusReader):
         Construct a new wordnet corpus reader, with the given root
         directory.
         """
+        if omw_reader is None:
+            warnings.warn(
+                "The multilingual functions are not available with this Wordnet version"
+            )
+
         super().__init__(root, self._FILES, encoding=self._ENCODING)
 
         # A index that provides the file offset
@@ -1287,8 +1293,9 @@ class WordNetCorpusReader(CorpusReader):
 
     def get_version(self):
         fh = self._data_file(ADJ)
+        fh.seek(0)
         for line in fh:
-            match = re.search(r"WordNet (\d+\.\d+) Copyright", line)
+            match = re.search(r"Word[nN]et (\d+|\d+\.\d+) Copyright", line)
             if match is not None:
                 version = match.group(1)
                 fh.seek(0)
@@ -1416,6 +1423,7 @@ class WordNetCorpusReader(CorpusReader):
             raise WordNetError(
                 f"No WordNet synset found for pos={pos} at offset={offset}."
             )
+        data_file.seek(0)
         return synset
 
     @deprecated("Use public method synset_from_pos_and_offset() instead")
diff --git a/nltk/metrics/agreement.py b/nltk/metrics/agreement.py
index 6c2c3f2e0..131bd5690 100644
--- a/nltk/metrics/agreement.py
+++ b/nltk/metrics/agreement.py
@@ -296,7 +296,7 @@ class AnnotationTask:
         if len(self.K) == 0:
             raise ValueError("Cannot calculate alpha, no data present!")
         if len(self.K) == 1:
-            log.debug("Only one annotation value, allpha returning 1.")
+            log.debug("Only one annotation value, alpha returning 1.")
             return 1
         if len(self.C) == 1 and len(self.I) == 1:
             raise ValueError("Cannot calculate alpha, only one coder and item present!")
diff --git a/nltk/metrics/distance.py b/nltk/metrics/distance.py
index 1f3211bd0..c5af07cb3 100644
--- a/nltk/metrics/distance.py
+++ b/nltk/metrics/distance.py
@@ -99,14 +99,19 @@ def edit_distance(s1, s2, substitution_cost=1, transpositions=False):
     last_left_t = _last_left_t_init(sigma)
 
     # iterate over the array
-    for i in range(len1):
-        last_right = 0
-        for j in range(len2):
-            last_left = last_left_t[s2[j]]
+    # i and j start from 1 and not 0 to stay close to the wikipedia pseudo-code
+    # see https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance
+    for i in range(1, len1 + 1):
+        last_right_buf = 0
+        for j in range(1, len2 + 1):
+            last_left = last_left_t[s2[j - 1]]
+            last_right = last_right_buf
+            if s1[i - 1] == s2[j - 1]:
+                last_right_buf = j
             _edit_dist_step(
                 lev,
-                i + 1,
-                j + 1,
+                i,
+                j,
                 s1,
                 s2,
                 last_left,
@@ -114,9 +119,7 @@ def edit_distance(s1, s2, substitution_cost=1, transpositions=False):
                 substitution_cost=substitution_cost,
                 transpositions=transpositions,
             )
-            if s1[i] == s2[j]:
-                last_right = j + 1
-            last_left_t[s1[i]] = i + 1
+        last_left_t[s1[i - 1]] = i
     return lev[len1][len2]
 
 
diff --git a/nltk/text.py b/nltk/text.py
index 3e3388e53..8ab8b2acc 100644
--- a/nltk/text.py
+++ b/nltk/text.py
@@ -608,7 +608,7 @@ class Text:
         See documentation for FreqDist.plot()
         :seealso: nltk.prob.FreqDist.plot()
         """
-        self.vocab().plot(*args)
+        return self.vocab().plot(*args)
 
     def vocab(self):
         """
diff --git a/nltk/tokenize/__init__.py b/nltk/tokenize/__init__.py
index 9a82830e1..e79fc6a47 100644
--- a/nltk/tokenize/__init__.py
+++ b/nltk/tokenize/__init__.py
@@ -88,7 +88,7 @@ from nltk.tokenize.sonority_sequencing import SyllableTokenizer
 from nltk.tokenize.stanford_segmenter import StanfordSegmenter
 from nltk.tokenize.texttiling import TextTilingTokenizer
 from nltk.tokenize.toktok import ToktokTokenizer
-from nltk.tokenize.treebank import TreebankWordTokenizer
+from nltk.tokenize.treebank import TreebankWordDetokenizer, TreebankWordTokenizer
 from nltk.tokenize.util import regexp_span_tokenize, string_span_tokenize
 
 
diff --git a/nltk/tokenize/destructive.py b/nltk/tokenize/destructive.py
index a4daf0941..32eb64fd8 100644
--- a/nltk/tokenize/destructive.py
+++ b/nltk/tokenize/destructive.py
@@ -1,7 +1,8 @@
 # Natural Language Toolkit: NLTK's very own tokenizer.
 #
 # Copyright (C) 2001-2021 NLTK Project
-# Author:
+# Author: Liling Tan
+#         Tom Aarsen <> (modifications)
 # URL: <https://www.nltk.org>
 # For license information, see LICENSE.TXT
 
@@ -9,6 +10,7 @@
 import re
 
 from nltk.tokenize.api import TokenizerI
+from nltk.tokenize.util import align_tokens
 
 
 class MacIntyreContractions:
@@ -24,7 +26,7 @@ class MacIntyreContractions:
         r"(?i)\b(got)(?#X)(ta)\b",
         r"(?i)\b(lem)(?#X)(me)\b",
         r"(?i)\b(more)(?#X)('n)\b",
-        r"(?i)\b(wan)(?#X)(na)\s",
+        r"(?i)\b(wan)(?#X)(na)(?=\s)",
     ]
     CONTRACTIONS3 = [r"(?i) ('t)(?#X)(is)\b", r"(?i) ('t)(?#X)(was)\b"]
     CONTRACTIONS4 = [r"(?i)\b(whad)(dd)(ya)\b", r"(?i)\b(wha)(t)(cha)\b"]
@@ -53,8 +55,8 @@ class NLTKWordTokenizer(TokenizerI):
     # Ending quotes.
     ENDING_QUOTES = [
         (re.compile("([»”’])", re.U), r" \1 "),
+        (re.compile(r"''"), " '' "),
         (re.compile(r'"'), " '' "),
-        (re.compile(r"(\S)(\'\')"), r"\1 \2 "),
         (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
         (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
     ]
@@ -146,3 +148,55 @@ class NLTKWordTokenizer(TokenizerI):
         #     text = regexp.sub(r' \1 \2 \3 ', text)
 
         return text if return_str else text.split()
+
+    def span_tokenize(self, text):
+        r"""
+        Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
+            >>> from nltk.tokenize import NLTKWordTokenizer
+            >>> s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
+            >>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
+            ... (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
+            ... (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
+            ... (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+            >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
+            True
+            >>> expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+            ... 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
+            ... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+            >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
+            True
+            Additional example
+            >>> from nltk.tokenize import NLTKWordTokenizer
+            >>> s = '''I said, "I'd like to buy some ''good muffins" which cost $3.88\n each in New (York)."'''
+            >>> expected = [(0, 1), (2, 6), (6, 7), (8, 9), (9, 10), (10, 12),
+            ... (13, 17), (18, 20), (21, 24), (25, 29), (30, 32), (32, 36),
+            ... (37, 44), (44, 45), (46, 51), (52, 56), (57, 58), (58, 62),
+            ... (64, 68), (69, 71), (72, 75), (76, 77), (77, 81), (81, 82),
+            ... (82, 83), (83, 84)]
+            >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
+            True
+            >>> expected = ['I', 'said', ',', '"', 'I', "'d", 'like', 'to',
+            ... 'buy', 'some', "''", "good", 'muffins', '"', 'which', 'cost',
+            ... '$', '3.88', 'each', 'in', 'New', '(', 'York', ')', '.', '"']
+            >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
+            True
+        """
+        raw_tokens = self.tokenize(text)
+
+        # Convert converted quotes back to original double quotes
+        # Do this only if original text contains double quote(s) or double
+        # single-quotes (because '' might be transformed to `` if it is
+        # treated as starting quotes).
+        if ('"' in text) or ("''" in text):
+            # Find double quotes and converted quotes
+            matched = [m.group() for m in re.finditer(r"``|'{2}|\"", text)]
+
+            # Replace converted quotes back to double quotes
+            tokens = [
+                matched.pop(0) if tok in ['"', "``", "''"] else tok
+                for tok in raw_tokens
+            ]
+        else:
+            tokens = raw_tokens
+
+        yield from align_tokens(tokens, text)
diff --git a/nltk/tokenize/treebank.py b/nltk/tokenize/treebank.py
index f5b90f5a0..51c202013 100644
--- a/nltk/tokenize/treebank.py
+++ b/nltk/tokenize/treebank.py
@@ -3,6 +3,7 @@
 # Copyright (C) 2001-2021 NLTK Project
 # Author: Edward Loper <edloper@gmail.com>
 #         Michael Heilman <mheilman@cmu.edu> (re-port from http://www.cis.upenn.edu/~treebank/tokenizer.sed)
+#         Tom Aarsen <> (modifications)
 #
 # URL: <https://www.nltk.org>
 # For license information, see LICENSE.TXT
@@ -86,8 +87,8 @@ class TreebankWordTokenizer(TokenizerI):
 
     # ending quotes
     ENDING_QUOTES = [
+        (re.compile(r"''"), " '' "),
         (re.compile(r'"'), " '' "),
-        (re.compile(r"(\S)(\'\')"), r"\1 \2 "),
         (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
         (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
     ]
diff --git a/nltk/tree/__init__.py b/nltk/tree/__init__.py
new file mode 100644
index 000000000..9b052ac6d
--- /dev/null
+++ b/nltk/tree/__init__.py
@@ -0,0 +1,35 @@
+# Natural Language Toolkit: Machine Translation
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+NLTK Tree Package
+
+This package may be used for representing hierarchical language
+structures, such as syntax trees and morphological trees.
+"""
+
+# TODO: add LabelledTree (can be used for dependency trees)
+
+from nltk.tree.immutable import (
+    ImmutableMultiParentedTree,
+    ImmutableParentedTree,
+    ImmutableProbabilisticTree,
+    ImmutableTree,
+)
+from nltk.tree.parented import MultiParentedTree, ParentedTree
+from nltk.tree.parsing import bracket_parse, sinica_parse
+from nltk.tree.prettyprinter import TreePrettyPrinter
+from nltk.tree.probabilistic import ProbabilisticTree
+from nltk.tree.transforms import (
+    chomsky_normal_form,
+    collapse_unary,
+    un_chomsky_normal_form,
+)
+from nltk.tree.tree import Tree
diff --git a/nltk/tree/immutable.py b/nltk/tree/immutable.py
new file mode 100644
index 000000000..072193927
--- /dev/null
+++ b/nltk/tree/immutable.py
@@ -0,0 +1,124 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.probability import ProbabilisticMixIn
+from nltk.tree.parented import MultiParentedTree, ParentedTree
+from nltk.tree.tree import Tree
+
+
+class ImmutableTree(Tree):
+    def __init__(self, node, children=None):
+        super().__init__(node, children)
+        # Precompute our hash value.  This ensures that we're really
+        # immutable.  It also means we only have to calculate it once.
+        try:
+            self._hash = hash((self._label, tuple(self)))
+        except (TypeError, ValueError) as e:
+            raise ValueError(
+                "%s: node value and children " "must be immutable" % type(self).__name__
+            ) from e
+
+    def __setitem__(self, index, value):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __setslice__(self, i, j, value):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __delitem__(self, index):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __delslice__(self, i, j):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __iadd__(self, other):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __imul__(self, other):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def append(self, v):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def extend(self, v):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def pop(self, v=None):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def remove(self, v):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def reverse(self):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def sort(self):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __hash__(self):
+        return self._hash
+
+    def set_label(self, value):
+        """
+        Set the node label.  This will only succeed the first time the
+        node label is set, which should occur in ImmutableTree.__init__().
+        """
+        if hasattr(self, "_label"):
+            raise ValueError("%s may not be modified" % type(self).__name__)
+        self._label = value
+
+
+class ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn):
+    def __init__(self, node, children=None, **prob_kwargs):
+        ImmutableTree.__init__(self, node, children)
+        ProbabilisticMixIn.__init__(self, **prob_kwargs)
+        self._hash = hash((self._label, tuple(self), self.prob()))
+
+    # We have to patch up these methods to make them work right:
+    def _frozen_class(self):
+        return ImmutableProbabilisticTree
+
+    def __repr__(self):
+        return f"{Tree.__repr__(self)} [{self.prob()}]"
+
+    def __str__(self):
+        return f"{self.pformat(margin=60)} [{self.prob()}]"
+
+    def copy(self, deep=False):
+        if not deep:
+            return type(self)(self._label, self, prob=self.prob())
+        else:
+            return type(self).convert(self)
+
+    @classmethod
+    def convert(cls, val):
+        if isinstance(val, Tree):
+            children = [cls.convert(child) for child in val]
+            if isinstance(val, ProbabilisticMixIn):
+                return cls(val._label, children, prob=val.prob())
+            else:
+                return cls(val._label, children, prob=1.0)
+        else:
+            return val
+
+
+class ImmutableParentedTree(ImmutableTree, ParentedTree):
+    pass
+
+
+class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree):
+    pass
+
+
+__all__ = [
+    "ImmutableProbabilisticTree",
+    "ImmutableTree",
+    "ImmutableParentedTree",
+    "ImmutableMultiParentedTree",
+]
diff --git a/nltk/tree/parented.py b/nltk/tree/parented.py
new file mode 100644
index 000000000..31955ec0d
--- /dev/null
+++ b/nltk/tree/parented.py
@@ -0,0 +1,590 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import warnings
+from abc import ABCMeta, abstractmethod
+
+from nltk.tree.tree import Tree
+from nltk.util import slice_bounds
+
+
+######################################################################
+## Parented trees
+######################################################################
+class AbstractParentedTree(Tree, metaclass=ABCMeta):
+    """
+    An abstract base class for a ``Tree`` that automatically maintains
+    pointers to parent nodes.  These parent pointers are updated
+    whenever any change is made to a tree's structure.  Two subclasses
+    are currently defined:
+
+      - ``ParentedTree`` is used for tree structures where each subtree
+        has at most one parent.  This class should be used in cases
+        where there is no"sharing" of subtrees.
+
+      - ``MultiParentedTree`` is used for tree structures where a
+        subtree may have zero or more parents.  This class should be
+        used in cases where subtrees may be shared.
+
+    Subclassing
+    ===========
+    The ``AbstractParentedTree`` class redefines all operations that
+    modify a tree's structure to call two methods, which are used by
+    subclasses to update parent information:
+
+      - ``_setparent()`` is called whenever a new child is added.
+      - ``_delparent()`` is called whenever a child is removed.
+    """
+
+    def __init__(self, node, children=None):
+        super().__init__(node, children)
+        # If children is None, the tree is read from node, and
+        # all parents will be set during parsing.
+        if children is not None:
+            # Otherwise we have to set the parent of the children.
+            # Iterate over self, and *not* children, because children
+            # might be an iterator.
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    self._setparent(child, i, dry_run=True)
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    self._setparent(child, i)
+
+    # ////////////////////////////////////////////////////////////
+    # Parent management
+    # ////////////////////////////////////////////////////////////
+    @abstractmethod
+    def _setparent(self, child, index, dry_run=False):
+        """
+        Update the parent pointer of ``child`` to point to ``self``.  This
+        method is only called if the type of ``child`` is ``Tree``;
+        i.e., it is not called when adding a leaf to a tree.  This method
+        is always called before the child is actually added to the
+        child list of ``self``.
+
+        :type child: Tree
+        :type index: int
+        :param index: The index of ``child`` in ``self``.
+        :raise TypeError: If ``child`` is a tree with an impropriate
+            type.  Typically, if ``child`` is a tree, then its type needs
+            to match the type of ``self``.  This prevents mixing of
+            different tree types (single-parented, multi-parented, and
+            non-parented).
+        :param dry_run: If true, the don't actually set the child's
+            parent pointer; just check for any error conditions, and
+            raise an exception if one is found.
+        """
+
+    @abstractmethod
+    def _delparent(self, child, index):
+        """
+        Update the parent pointer of ``child`` to not point to self.  This
+        method is only called if the type of ``child`` is ``Tree``; i.e., it
+        is not called when removing a leaf from a tree.  This method
+        is always called before the child is actually removed from the
+        child list of ``self``.
+
+        :type child: Tree
+        :type index: int
+        :param index: The index of ``child`` in ``self``.
+        """
+
+    # ////////////////////////////////////////////////////////////
+    # Methods that add/remove children
+    # ////////////////////////////////////////////////////////////
+    # Every method that adds or removes a child must make
+    # appropriate calls to _setparent() and _delparent().
+
+    def __delitem__(self, index):
+        # del ptree[start:stop]
+        if isinstance(index, slice):
+            start, stop, step = slice_bounds(self, index, allow_step=True)
+            # Clear all the children pointers.
+            for i in range(start, stop, step):
+                if isinstance(self[i], Tree):
+                    self._delparent(self[i], i)
+            # Delete the children from our child list.
+            super().__delitem__(index)
+
+        # del ptree[i]
+        elif isinstance(index, int):
+            if index < 0:
+                index += len(self)
+            if index < 0:
+                raise IndexError("index out of range")
+            # Clear the child's parent pointer.
+            if isinstance(self[index], Tree):
+                self._delparent(self[index], index)
+            # Remove the child from our child list.
+            super().__delitem__(index)
+
+        elif isinstance(index, (list, tuple)):
+            # del ptree[()]
+            if len(index) == 0:
+                raise IndexError("The tree position () may not be deleted.")
+            # del ptree[(i,)]
+            elif len(index) == 1:
+                del self[index[0]]
+            # del ptree[i1, i2, i3]
+            else:
+                del self[index[0]][index[1:]]
+
+        else:
+            raise TypeError(
+                "%s indices must be integers, not %s"
+                % (type(self).__name__, type(index).__name__)
+            )
+
+    def __setitem__(self, index, value):
+        # ptree[start:stop] = value
+        if isinstance(index, slice):
+            start, stop, step = slice_bounds(self, index, allow_step=True)
+            # make a copy of value, in case it's an iterator
+            if not isinstance(value, (list, tuple)):
+                value = list(value)
+            # Check for any error conditions, so we can avoid ending
+            # up in an inconsistent state if an error does occur.
+            for i, child in enumerate(value):
+                if isinstance(child, Tree):
+                    self._setparent(child, start + i * step, dry_run=True)
+            # clear the child pointers of all parents we're removing
+            for i in range(start, stop, step):
+                if isinstance(self[i], Tree):
+                    self._delparent(self[i], i)
+            # set the child pointers of the new children.  We do this
+            # after clearing *all* child pointers, in case we're e.g.
+            # reversing the elements in a tree.
+            for i, child in enumerate(value):
+                if isinstance(child, Tree):
+                    self._setparent(child, start + i * step)
+            # finally, update the content of the child list itself.
+            super().__setitem__(index, value)
+
+        # ptree[i] = value
+        elif isinstance(index, int):
+            if index < 0:
+                index += len(self)
+            if index < 0:
+                raise IndexError("index out of range")
+            # if the value is not changing, do nothing.
+            if value is self[index]:
+                return
+            # Set the new child's parent pointer.
+            if isinstance(value, Tree):
+                self._setparent(value, index)
+            # Remove the old child's parent pointer
+            if isinstance(self[index], Tree):
+                self._delparent(self[index], index)
+            # Update our child list.
+            super().__setitem__(index, value)
+
+        elif isinstance(index, (list, tuple)):
+            # ptree[()] = value
+            if len(index) == 0:
+                raise IndexError("The tree position () may not be assigned to.")
+            # ptree[(i,)] = value
+            elif len(index) == 1:
+                self[index[0]] = value
+            # ptree[i1, i2, i3] = value
+            else:
+                self[index[0]][index[1:]] = value
+
+        else:
+            raise TypeError(
+                "%s indices must be integers, not %s"
+                % (type(self).__name__, type(index).__name__)
+            )
+
+    def append(self, child):
+        if isinstance(child, Tree):
+            self._setparent(child, len(self))
+        super().append(child)
+
+    def extend(self, children):
+        for child in children:
+            if isinstance(child, Tree):
+                self._setparent(child, len(self))
+            super().append(child)
+
+    def insert(self, index, child):
+        # Handle negative indexes.  Note that if index < -len(self),
+        # we do *not* raise an IndexError, unlike __getitem__.  This
+        # is done for consistency with list.__getitem__ and list.index.
+        if index < 0:
+            index += len(self)
+        if index < 0:
+            index = 0
+        # Set the child's parent, and update our child list.
+        if isinstance(child, Tree):
+            self._setparent(child, index)
+        super().insert(index, child)
+
+    def pop(self, index=-1):
+        if index < 0:
+            index += len(self)
+        if index < 0:
+            raise IndexError("index out of range")
+        if isinstance(self[index], Tree):
+            self._delparent(self[index], index)
+        return super().pop(index)
+
+    # n.b.: like `list`, this is done by equality, not identity!
+    # To remove a specific child, use del ptree[i].
+    def remove(self, child):
+        index = self.index(child)
+        if isinstance(self[index], Tree):
+            self._delparent(self[index], index)
+        super().remove(child)
+
+    # We need to implement __getslice__ and friends, even though
+    # they're deprecated, because otherwise list.__getslice__ will get
+    # called (since we're subclassing from list).  Just delegate to
+    # __getitem__ etc., but use max(0, start) and max(0, stop) because
+    # because negative indices are already handled *before*
+    # __getslice__ is called; and we don't want to double-count them.
+    if hasattr(list, "__getslice__"):
+
+        def __getslice__(self, start, stop):
+            return self.__getitem__(slice(max(0, start), max(0, stop)))
+
+        def __delslice__(self, start, stop):
+            return self.__delitem__(slice(max(0, start), max(0, stop)))
+
+        def __setslice__(self, start, stop, value):
+            return self.__setitem__(slice(max(0, start), max(0, stop)), value)
+
+    def __getnewargs__(self):
+        """Method used by the pickle module when un-pickling.
+        This method provides the arguments passed to ``__new__``
+        upon un-pickling. Without this method, ParentedTree instances
+        cannot be pickled and unpickled in Python 3.7+ onwards.
+
+        :return: Tuple of arguments for ``__new__``, i.e. the label
+            and the children of this node.
+        :rtype: Tuple[Any, List[AbstractParentedTree]]
+        """
+        return (self._label, list(self))
+
+
+class ParentedTree(AbstractParentedTree):
+    """
+    A ``Tree`` that automatically maintains parent pointers for
+    single-parented trees.  The following are methods for querying
+    the structure of a parented tree: ``parent``, ``parent_index``,
+    ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``.
+
+    Each ``ParentedTree`` may have at most one parent.  In
+    particular, subtrees may not be shared.  Any attempt to reuse a
+    single ``ParentedTree`` as a child of more than one parent (or
+    as multiple children of the same parent) will cause a
+    ``ValueError`` exception to be raised.
+
+    ``ParentedTrees`` should never be used in the same tree as ``Trees``
+    or ``MultiParentedTrees``.  Mixing tree implementations may result
+    in incorrect parent pointers and in ``TypeError`` exceptions.
+    """
+
+    def __init__(self, node, children=None):
+        self._parent = None
+        """The parent of this Tree, or None if it has no parent."""
+        super().__init__(node, children)
+        if children is None:
+            # If children is None, the tree is read from node.
+            # After parsing, the parent of the immediate children
+            # will point to an intermediate tree, not self.
+            # We fix this by brute force:
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    child._parent = None
+                    self._setparent(child, i)
+
+    def _frozen_class(self):
+        from nltk.tree.immutable import ImmutableParentedTree
+
+        return ImmutableParentedTree
+
+    def copy(self, deep=False):
+        if not deep:
+            warnings.warn(
+                f"{self.__class__.__name__} objects do not support shallow copies. Defaulting to a deep copy."
+            )
+        return super().copy(deep=True)
+
+    # /////////////////////////////////////////////////////////////////
+    # Methods
+    # /////////////////////////////////////////////////////////////////
+
+    def parent(self):
+        """The parent of this tree, or None if it has no parent."""
+        return self._parent
+
+    def parent_index(self):
+        """
+        The index of this tree in its parent.  I.e.,
+        ``ptree.parent()[ptree.parent_index()] is ptree``.  Note that
+        ``ptree.parent_index()`` is not necessarily equal to
+        ``ptree.parent.index(ptree)``, since the ``index()`` method
+        returns the first child that is equal to its argument.
+        """
+        if self._parent is None:
+            return None
+        for i, child in enumerate(self._parent):
+            if child is self:
+                return i
+        assert False, "expected to find self in self._parent!"
+
+    def left_sibling(self):
+        """The left sibling of this tree, or None if it has none."""
+        parent_index = self.parent_index()
+        if self._parent and parent_index > 0:
+            return self._parent[parent_index - 1]
+        return None  # no left sibling
+
+    def right_sibling(self):
+        """The right sibling of this tree, or None if it has none."""
+        parent_index = self.parent_index()
+        if self._parent and parent_index < (len(self._parent) - 1):
+            return self._parent[parent_index + 1]
+        return None  # no right sibling
+
+    def root(self):
+        """
+        The root of this tree.  I.e., the unique ancestor of this tree
+        whose parent is None.  If ``ptree.parent()`` is None, then
+        ``ptree`` is its own root.
+        """
+        root = self
+        while root.parent() is not None:
+            root = root.parent()
+        return root
+
+    def treeposition(self):
+        """
+        The tree position of this tree, relative to the root of the
+        tree.  I.e., ``ptree.root[ptree.treeposition] is ptree``.
+        """
+        if self.parent() is None:
+            return ()
+        else:
+            return self.parent().treeposition() + (self.parent_index(),)
+
+    # /////////////////////////////////////////////////////////////////
+    # Parent Management
+    # /////////////////////////////////////////////////////////////////
+
+    def _delparent(self, child, index):
+        # Sanity checks
+        assert isinstance(child, ParentedTree)
+        assert self[index] is child
+        assert child._parent is self
+
+        # Delete child's parent pointer.
+        child._parent = None
+
+    def _setparent(self, child, index, dry_run=False):
+        # If the child's type is incorrect, then complain.
+        if not isinstance(child, ParentedTree):
+            raise TypeError("Can not insert a non-ParentedTree into a ParentedTree")
+
+        # If child already has a parent, then complain.
+        if hasattr(child, "_parent") and child._parent is not None:
+            raise ValueError("Can not insert a subtree that already has a parent.")
+
+        # Set child's parent pointer & index.
+        if not dry_run:
+            child._parent = self
+
+
+class MultiParentedTree(AbstractParentedTree):
+    """
+    A ``Tree`` that automatically maintains parent pointers for
+    multi-parented trees.  The following are methods for querying the
+    structure of a multi-parented tree: ``parents()``, ``parent_indices()``,
+    ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``.
+
+    Each ``MultiParentedTree`` may have zero or more parents.  In
+    particular, subtrees may be shared.  If a single
+    ``MultiParentedTree`` is used as multiple children of the same
+    parent, then that parent will appear multiple times in its
+    ``parents()`` method.
+
+    ``MultiParentedTrees`` should never be used in the same tree as
+    ``Trees`` or ``ParentedTrees``.  Mixing tree implementations may
+    result in incorrect parent pointers and in ``TypeError`` exceptions.
+    """
+
+    def __init__(self, node, children=None):
+        self._parents = []
+        """A list of this tree's parents.  This list should not
+           contain duplicates, even if a parent contains this tree
+           multiple times."""
+        super().__init__(node, children)
+        if children is None:
+            # If children is None, the tree is read from node.
+            # After parsing, the parent(s) of the immediate children
+            # will point to an intermediate tree, not self.
+            # We fix this by brute force:
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    child._parents = []
+                    self._setparent(child, i)
+
+    def _frozen_class(self):
+        from nltk.tree.immutable import ImmutableMultiParentedTree
+
+        return ImmutableMultiParentedTree
+
+    # /////////////////////////////////////////////////////////////////
+    # Methods
+    # /////////////////////////////////////////////////////////////////
+
+    def parents(self):
+        """
+        The set of parents of this tree.  If this tree has no parents,
+        then ``parents`` is the empty set.  To check if a tree is used
+        as multiple children of the same parent, use the
+        ``parent_indices()`` method.
+
+        :type: list(MultiParentedTree)
+        """
+        return list(self._parents)
+
+    def left_siblings(self):
+        """
+        A list of all left siblings of this tree, in any of its parent
+        trees.  A tree may be its own left sibling if it is used as
+        multiple contiguous children of the same parent.  A tree may
+        appear multiple times in this list if it is the left sibling
+        of this tree with respect to multiple parents.
+
+        :type: list(MultiParentedTree)
+        """
+        return [
+            parent[index - 1]
+            for (parent, index) in self._get_parent_indices()
+            if index > 0
+        ]
+
+    def right_siblings(self):
+        """
+        A list of all right siblings of this tree, in any of its parent
+        trees.  A tree may be its own right sibling if it is used as
+        multiple contiguous children of the same parent.  A tree may
+        appear multiple times in this list if it is the right sibling
+        of this tree with respect to multiple parents.
+
+        :type: list(MultiParentedTree)
+        """
+        return [
+            parent[index + 1]
+            for (parent, index) in self._get_parent_indices()
+            if index < (len(parent) - 1)
+        ]
+
+    def _get_parent_indices(self):
+        return [
+            (parent, index)
+            for parent in self._parents
+            for index, child in enumerate(parent)
+            if child is self
+        ]
+
+    def roots(self):
+        """
+        The set of all roots of this tree.  This set is formed by
+        tracing all possible parent paths until trees with no parents
+        are found.
+
+        :type: list(MultiParentedTree)
+        """
+        return list(self._get_roots_helper({}).values())
+
+    def _get_roots_helper(self, result):
+        if self._parents:
+            for parent in self._parents:
+                parent._get_roots_helper(result)
+        else:
+            result[id(self)] = self
+        return result
+
+    def parent_indices(self, parent):
+        """
+        Return a list of the indices where this tree occurs as a child
+        of ``parent``.  If this child does not occur as a child of
+        ``parent``, then the empty list is returned.  The following is
+        always true::
+
+          for parent_index in ptree.parent_indices(parent):
+              parent[parent_index] is ptree
+        """
+        if parent not in self._parents:
+            return []
+        else:
+            return [index for (index, child) in enumerate(parent) if child is self]
+
+    def treepositions(self, root):
+        """
+        Return a list of all tree positions that can be used to reach
+        this multi-parented tree starting from ``root``.  I.e., the
+        following is always true::
+
+          for treepos in ptree.treepositions(root):
+              root[treepos] is ptree
+        """
+        if self is root:
+            return [()]
+        else:
+            return [
+                treepos + (index,)
+                for parent in self._parents
+                for treepos in parent.treepositions(root)
+                for (index, child) in enumerate(parent)
+                if child is self
+            ]
+
+    # /////////////////////////////////////////////////////////////////
+    # Parent Management
+    # /////////////////////////////////////////////////////////////////
+
+    def _delparent(self, child, index):
+        # Sanity checks
+        assert isinstance(child, MultiParentedTree)
+        assert self[index] is child
+        assert len([p for p in child._parents if p is self]) == 1
+
+        # If the only copy of child in self is at index, then delete
+        # self from child's parent list.
+        for i, c in enumerate(self):
+            if c is child and i != index:
+                break
+        else:
+            child._parents.remove(self)
+
+    def _setparent(self, child, index, dry_run=False):
+        # If the child's type is incorrect, then complain.
+        if not isinstance(child, MultiParentedTree):
+            raise TypeError(
+                "Can not insert a non-MultiParentedTree into a MultiParentedTree"
+            )
+
+        # Add self as a parent pointer if it's not already listed.
+        if not dry_run:
+            for parent in child._parents:
+                if parent is self:
+                    break
+            else:
+                child._parents.append(self)
+
+
+__all__ = [
+    "ParentedTree",
+    "MultiParentedTree",
+]
diff --git a/nltk/tree/parsing.py b/nltk/tree/parsing.py
new file mode 100644
index 000000000..062662233
--- /dev/null
+++ b/nltk/tree/parsing.py
@@ -0,0 +1,66 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+
+from nltk.tree.tree import Tree
+
+######################################################################
+## Parsing
+######################################################################
+
+
+def bracket_parse(s):
+    """
+    Use Tree.read(s, remove_empty_top_bracketing=True) instead.
+    """
+    raise NameError("Use Tree.read(s, remove_empty_top_bracketing=True) instead.")
+
+
+def sinica_parse(s):
+    """
+    Parse a Sinica Treebank string and return a tree.  Trees are represented as nested brackettings,
+    as shown in the following example (X represents a Chinese character):
+    S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY)
+
+    :return: A tree corresponding to the string representation.
+    :rtype: Tree
+    :param s: The string to be converted
+    :type s: str
+    """
+    tokens = re.split(r"([()| ])", s)
+    for i in range(len(tokens)):
+        if tokens[i] == "(":
+            tokens[i - 1], tokens[i] = (
+                tokens[i],
+                tokens[i - 1],
+            )  # pull nonterminal inside parens
+        elif ":" in tokens[i]:
+            fields = tokens[i].split(":")
+            if len(fields) == 2:  # non-terminal
+                tokens[i] = fields[1]
+            else:
+                tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")"
+        elif tokens[i] == "|":
+            tokens[i] = ""
+
+    treebank_string = " ".join(tokens)
+    return Tree.fromstring(treebank_string, remove_empty_top_bracketing=True)
+
+
+#    s = re.sub(r'^#[^\s]*\s', '', s)  # remove leading identifier
+#    s = re.sub(r'\w+:', '', s)       # remove role tags
+
+#    return s
+
+__all__ = [
+    "bracket_parse",
+    "sinica_parse",
+]
diff --git a/nltk/tree/prettyprinter.py b/nltk/tree/prettyprinter.py
new file mode 100644
index 000000000..2ce106e05
--- /dev/null
+++ b/nltk/tree/prettyprinter.py
@@ -0,0 +1,628 @@
+# Natural Language Toolkit: ASCII visualization of NLTK trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Andreas van Cranenburgh <A.W.vanCranenburgh@uva.nl>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Pretty-printing of discontinuous trees.
+Adapted from the disco-dop project, by Andreas van Cranenburgh.
+https://github.com/andreasvc/disco-dop
+
+Interesting reference (not used for this code):
+T. Eschbach et al., Orth. Hypergraph Drawing, Journal of
+Graph Algorithms and Applications, 10(2) 141--157 (2006)149.
+https://jgaa.info/accepted/2006/EschbachGuentherBecker2006.10.2.pdf
+"""
+
+import re
+
+try:
+    from html import escape
+except ImportError:
+    from cgi import escape
+
+from collections import defaultdict
+from operator import itemgetter
+
+from nltk.tree.tree import Tree
+from nltk.util import OrderedDict
+
+ANSICOLOR = {
+    "black": 30,
+    "red": 31,
+    "green": 32,
+    "yellow": 33,
+    "blue": 34,
+    "magenta": 35,
+    "cyan": 36,
+    "white": 37,
+}
+
+
+class TreePrettyPrinter:
+    """
+    Pretty-print a tree in text format, either as ASCII or Unicode.
+    The tree can be a normal tree, or discontinuous.
+
+    ``TreePrettyPrinter(tree, sentence=None, highlight=())``
+    creates an object from which different visualizations can be created.
+
+    :param tree: a Tree object.
+    :param sentence: a list of words (strings). If `sentence` is given,
+        `tree` must contain integers as leaves, which are taken as indices
+        in `sentence`. Using this you can display a discontinuous tree.
+    :param highlight: Optionally, a sequence of Tree objects in `tree` which
+        should be highlighted. Has the effect of only applying colors to nodes
+        in this sequence (nodes should be given as Tree objects, terminals as
+        indices).
+
+    >>> from nltk.tree import Tree
+    >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))')
+    >>> print(TreePrettyPrinter(tree).text())
+    ... # doctest: +NORMALIZE_WHITESPACE
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+    """
+
+    def __init__(self, tree, sentence=None, highlight=()):
+        if sentence is None:
+            leaves = tree.leaves()
+            if (
+                leaves
+                and not any(len(a) == 0 for a in tree.subtrees())
+                and all(isinstance(a, int) for a in leaves)
+            ):
+                sentence = [str(a) for a in leaves]
+            else:
+                # this deals with empty nodes (frontier non-terminals)
+                # and multiple/mixed terminals under non-terminals.
+                tree = tree.copy(True)
+                sentence = []
+                for a in tree.subtrees():
+                    if len(a) == 0:
+                        a.append(len(sentence))
+                        sentence.append(None)
+                    elif any(not isinstance(b, Tree) for b in a):
+                        for n, b in enumerate(a):
+                            if not isinstance(b, Tree):
+                                a[n] = len(sentence)
+                                if type(b) == tuple:
+                                    b = "/".join(b)
+                                sentence.append("%s" % b)
+        self.nodes, self.coords, self.edges, self.highlight = self.nodecoords(
+            tree, sentence, highlight
+        )
+
+    def __str__(self):
+        return self.text()
+
+    def __repr__(self):
+        return "<TreePrettyPrinter with %d nodes>" % len(self.nodes)
+
+    @staticmethod
+    def nodecoords(tree, sentence, highlight):
+        """
+        Produce coordinates of nodes on a grid.
+
+        Objective:
+
+        - Produce coordinates for a non-overlapping placement of nodes and
+            horizontal lines.
+        - Order edges so that crossing edges cross a minimal number of previous
+            horizontal lines (never vertical lines).
+
+        Approach:
+
+        - bottom up level order traversal (start at terminals)
+        - at each level, identify nodes which cannot be on the same row
+        - identify nodes which cannot be in the same column
+        - place nodes into a grid at (row, column)
+        - order child-parent edges with crossing edges last
+
+        Coordinates are (row, column); the origin (0, 0) is at the top left;
+        the root node is on row 0. Coordinates do not consider the size of a
+        node (which depends on font, &c), so the width of a column of the grid
+        should be automatically determined by the element with the greatest
+        width in that column. Alternatively, the integer coordinates could be
+        converted to coordinates in which the distances between adjacent nodes
+        are non-uniform.
+
+        Produces tuple (nodes, coords, edges, highlighted) where:
+
+        - nodes[id]: Tree object for the node with this integer id
+        - coords[id]: (n, m) coordinate where to draw node with id in the grid
+        - edges[id]: parent id of node with this id (ordered dictionary)
+        - highlighted: set of ids that should be highlighted
+        """
+
+        def findcell(m, matrix, startoflevel, children):
+            """
+            Find vacant row, column index for node ``m``.
+            Iterate over current rows for this level (try lowest first)
+            and look for cell between first and last child of this node,
+            add new row to level if no free row available.
+            """
+            candidates = [a for _, a in children[m]]
+            minidx, maxidx = min(candidates), max(candidates)
+            leaves = tree[m].leaves()
+            center = scale * sum(leaves) // len(leaves)  # center of gravity
+            if minidx < maxidx and not minidx < center < maxidx:
+                center = sum(candidates) // len(candidates)
+            if max(candidates) - min(candidates) > 2 * scale:
+                center -= center % scale  # round to unscaled coordinate
+                if minidx < maxidx and not minidx < center < maxidx:
+                    center += scale
+            if ids[m] == 0:
+                startoflevel = len(matrix)
+            for rowidx in range(startoflevel, len(matrix) + 1):
+                if rowidx == len(matrix):  # need to add a new row
+                    matrix.append(
+                        [
+                            vertline if a not in (corner, None) else None
+                            for a in matrix[-1]
+                        ]
+                    )
+                row = matrix[rowidx]
+                i = j = center
+                if len(children[m]) == 1:  # place unaries directly above child
+                    return rowidx, next(iter(children[m]))[1]
+                elif all(
+                    a is None or a == vertline
+                    for a in row[min(candidates) : max(candidates) + 1]
+                ):
+                    # find free column
+                    for n in range(scale):
+                        i = j = center + n
+                        while j > minidx or i < maxidx:
+                            if i < maxidx and (
+                                matrix[rowidx][i] is None or i in candidates
+                            ):
+                                return rowidx, i
+                            elif j > minidx and (
+                                matrix[rowidx][j] is None or j in candidates
+                            ):
+                                return rowidx, j
+                            i += scale
+                            j -= scale
+            raise ValueError(
+                "could not find a free cell for:\n%s\n%s"
+                "min=%d; max=%d" % (tree[m], minidx, maxidx, dumpmatrix())
+            )
+
+        def dumpmatrix():
+            """Dump matrix contents for debugging purposes."""
+            return "\n".join(
+                "%2d: %s" % (n, " ".join(("%2r" % i)[:2] for i in row))
+                for n, row in enumerate(matrix)
+            )
+
+        leaves = tree.leaves()
+        if not all(isinstance(n, int) for n in leaves):
+            raise ValueError("All leaves must be integer indices.")
+        if len(leaves) != len(set(leaves)):
+            raise ValueError("Indices must occur at most once.")
+        if not all(0 <= n < len(sentence) for n in leaves):
+            raise ValueError(
+                "All leaves must be in the interval 0..n "
+                "with n=len(sentence)\ntokens: %d indices: "
+                "%r\nsentence: %s" % (len(sentence), tree.leaves(), sentence)
+            )
+        vertline, corner = -1, -2  # constants
+        tree = tree.copy(True)
+        for a in tree.subtrees():
+            a.sort(key=lambda n: min(n.leaves()) if isinstance(n, Tree) else n)
+        scale = 2
+        crossed = set()
+        # internal nodes and lexical nodes (no frontiers)
+        positions = tree.treepositions()
+        maxdepth = max(map(len, positions)) + 1
+        childcols = defaultdict(set)
+        matrix = [[None] * (len(sentence) * scale)]
+        nodes = {}
+        ids = {a: n for n, a in enumerate(positions)}
+        highlighted_nodes = {
+            n for a, n in ids.items() if not highlight or tree[a] in highlight
+        }
+        levels = {n: [] for n in range(maxdepth - 1)}
+        terminals = []
+        for a in positions:
+            node = tree[a]
+            if isinstance(node, Tree):
+                levels[maxdepth - node.height()].append(a)
+            else:
+                terminals.append(a)
+
+        for n in levels:
+            levels[n].sort(key=lambda n: max(tree[n].leaves()) - min(tree[n].leaves()))
+        terminals.sort()
+        positions = set(positions)
+
+        for m in terminals:
+            i = int(tree[m]) * scale
+            assert matrix[0][i] is None, (matrix[0][i], m, i)
+            matrix[0][i] = ids[m]
+            nodes[ids[m]] = sentence[tree[m]]
+            if nodes[ids[m]] is None:
+                nodes[ids[m]] = "..."
+                highlighted_nodes.discard(ids[m])
+            positions.remove(m)
+            childcols[m[:-1]].add((0, i))
+
+        # add other nodes centered on their children,
+        # if the center is already taken, back off
+        # to the left and right alternately, until an empty cell is found.
+        for n in sorted(levels, reverse=True):
+            nodesatdepth = levels[n]
+            startoflevel = len(matrix)
+            matrix.append(
+                [vertline if a not in (corner, None) else None for a in matrix[-1]]
+            )
+            for m in nodesatdepth:  # [::-1]:
+                if n < maxdepth - 1 and childcols[m]:
+                    _, pivot = min(childcols[m], key=itemgetter(1))
+                    if {
+                        a[:-1]
+                        for row in matrix[:-1]
+                        for a in row[:pivot]
+                        if isinstance(a, tuple)
+                    } & {
+                        a[:-1]
+                        for row in matrix[:-1]
+                        for a in row[pivot:]
+                        if isinstance(a, tuple)
+                    }:
+                        crossed.add(m)
+
+                rowidx, i = findcell(m, matrix, startoflevel, childcols)
+                positions.remove(m)
+
+                # block positions where children of this node branch out
+                for _, x in childcols[m]:
+                    matrix[rowidx][x] = corner
+                # assert m == () or matrix[rowidx][i] in (None, corner), (
+                #         matrix[rowidx][i], m, str(tree), ' '.join(sentence))
+                # node itself
+                matrix[rowidx][i] = ids[m]
+                nodes[ids[m]] = tree[m]
+                # add column to the set of children for its parent
+                if m != ():
+                    childcols[m[:-1]].add((rowidx, i))
+        assert len(positions) == 0
+
+        # remove unused columns, right to left
+        for m in range(scale * len(sentence) - 1, -1, -1):
+            if not any(isinstance(row[m], (Tree, int)) for row in matrix):
+                for row in matrix:
+                    del row[m]
+
+        # remove unused rows, reverse
+        matrix = [
+            row
+            for row in reversed(matrix)
+            if not all(a is None or a == vertline for a in row)
+        ]
+
+        # collect coordinates of nodes
+        coords = {}
+        for n, _ in enumerate(matrix):
+            for m, i in enumerate(matrix[n]):
+                if isinstance(i, int) and i >= 0:
+                    coords[i] = n, m
+
+        # move crossed edges last
+        positions = sorted(
+            (a for level in levels.values() for a in level),
+            key=lambda a: a[:-1] in crossed,
+        )
+
+        # collect edges from node to node
+        edges = OrderedDict()
+        for i in reversed(positions):
+            for j, _ in enumerate(tree[i]):
+                edges[ids[i + (j,)]] = ids[i]
+
+        return nodes, coords, edges, highlighted_nodes
+
+    def text(
+        self,
+        nodedist=1,
+        unicodelines=False,
+        html=False,
+        ansi=False,
+        nodecolor="blue",
+        leafcolor="red",
+        funccolor="green",
+        abbreviate=None,
+        maxwidth=16,
+    ):
+        """
+        :return: ASCII art for a discontinuous tree.
+
+        :param unicodelines: whether to use Unicode line drawing characters
+            instead of plain (7-bit) ASCII.
+        :param html: whether to wrap output in html code (default plain text).
+        :param ansi: whether to produce colors with ANSI escape sequences
+            (only effective when html==False).
+        :param leafcolor, nodecolor: specify colors of leaves and phrasal
+            nodes; effective when either html or ansi is True.
+        :param abbreviate: if True, abbreviate labels longer than 5 characters.
+            If integer, abbreviate labels longer than `abbr` characters.
+        :param maxwidth: maximum number of characters before a label starts to
+            wrap; pass None to disable.
+        """
+        if abbreviate == True:
+            abbreviate = 5
+        if unicodelines:
+            horzline = "\u2500"
+            leftcorner = "\u250c"
+            rightcorner = "\u2510"
+            vertline = " \u2502 "
+            tee = horzline + "\u252C" + horzline
+            bottom = horzline + "\u2534" + horzline
+            cross = horzline + "\u253c" + horzline
+            ellipsis = "\u2026"
+        else:
+            horzline = "_"
+            leftcorner = rightcorner = " "
+            vertline = " | "
+            tee = 3 * horzline
+            cross = bottom = "_|_"
+            ellipsis = "."
+
+        def crosscell(cur, x=vertline):
+            """Overwrite center of this cell with a vertical branch."""
+            splitl = len(cur) - len(cur) // 2 - len(x) // 2 - 1
+            lst = list(cur)
+            lst[splitl : splitl + len(x)] = list(x)
+            return "".join(lst)
+
+        result = []
+        matrix = defaultdict(dict)
+        maxnodewith = defaultdict(lambda: 3)
+        maxnodeheight = defaultdict(lambda: 1)
+        maxcol = 0
+        minchildcol = {}
+        maxchildcol = {}
+        childcols = defaultdict(set)
+        labels = {}
+        wrapre = re.compile(
+            "(.{%d,%d}\\b\\W*|.{%d})" % (maxwidth - 4, maxwidth, maxwidth)
+        )
+        # collect labels and coordinates
+        for a in self.nodes:
+            row, column = self.coords[a]
+            matrix[row][column] = a
+            maxcol = max(maxcol, column)
+            label = (
+                self.nodes[a].label()
+                if isinstance(self.nodes[a], Tree)
+                else self.nodes[a]
+            )
+            if abbreviate and len(label) > abbreviate:
+                label = label[:abbreviate] + ellipsis
+            if maxwidth and len(label) > maxwidth:
+                label = wrapre.sub(r"\1\n", label).strip()
+            label = label.split("\n")
+            maxnodeheight[row] = max(maxnodeheight[row], len(label))
+            maxnodewith[column] = max(maxnodewith[column], max(map(len, label)))
+            labels[a] = label
+            if a not in self.edges:
+                continue  # e.g., root
+            parent = self.edges[a]
+            childcols[parent].add((row, column))
+            minchildcol[parent] = min(minchildcol.get(parent, column), column)
+            maxchildcol[parent] = max(maxchildcol.get(parent, column), column)
+        # bottom up level order traversal
+        for row in sorted(matrix, reverse=True):
+            noderows = [
+                ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
+                for _ in range(maxnodeheight[row])
+            ]
+            branchrow = ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
+            for col in matrix[row]:
+                n = matrix[row][col]
+                node = self.nodes[n]
+                text = labels[n]
+                if isinstance(node, Tree):
+                    # draw horizontal branch towards children for this node
+                    if n in minchildcol and minchildcol[n] < maxchildcol[n]:
+                        i, j = minchildcol[n], maxchildcol[n]
+                        a, b = (maxnodewith[i] + 1) // 2 - 1, maxnodewith[j] // 2
+                        branchrow[i] = ((" " * a) + leftcorner).ljust(
+                            maxnodewith[i], horzline
+                        )
+                        branchrow[j] = (rightcorner + (" " * b)).rjust(
+                            maxnodewith[j], horzline
+                        )
+                        for i in range(minchildcol[n] + 1, maxchildcol[n]):
+                            if i == col and any(a == i for _, a in childcols[n]):
+                                line = cross
+                            elif i == col:
+                                line = bottom
+                            elif any(a == i for _, a in childcols[n]):
+                                line = tee
+                            else:
+                                line = horzline
+                            branchrow[i] = line.center(maxnodewith[i], horzline)
+                    else:  # if n and n in minchildcol:
+                        branchrow[col] = crosscell(branchrow[col])
+                text = [a.center(maxnodewith[col]) for a in text]
+                color = nodecolor if isinstance(node, Tree) else leafcolor
+                if isinstance(node, Tree) and node.label().startswith("-"):
+                    color = funccolor
+                if html:
+                    text = [escape(a, quote=False) for a in text]
+                    if n in self.highlight:
+                        text = [f"<font color={color}>{a}</font>" for a in text]
+                elif ansi and n in self.highlight:
+                    text = ["\x1b[%d;1m%s\x1b[0m" % (ANSICOLOR[color], a) for a in text]
+                for x in range(maxnodeheight[row]):
+                    # draw vertical lines in partially filled multiline node
+                    # labels, but only if it's not a frontier node.
+                    noderows[x][col] = (
+                        text[x]
+                        if x < len(text)
+                        else (vertline if childcols[n] else " ").center(
+                            maxnodewith[col], " "
+                        )
+                    )
+            # for each column, if there is a node below us which has a parent
+            # above us, draw a vertical branch in that column.
+            if row != max(matrix):
+                for n, (childrow, col) in self.coords.items():
+                    if n > 0 and self.coords[self.edges[n]][0] < row < childrow:
+                        branchrow[col] = crosscell(branchrow[col])
+                        if col not in matrix[row]:
+                            for noderow in noderows:
+                                noderow[col] = crosscell(noderow[col])
+                branchrow = [
+                    a + ((a[-1] if a[-1] != " " else b[0]) * nodedist)
+                    for a, b in zip(branchrow, branchrow[1:] + [" "])
+                ]
+                result.append("".join(branchrow))
+            result.extend(
+                (" " * nodedist).join(noderow) for noderow in reversed(noderows)
+            )
+        return "\n".join(reversed(result)) + "\n"
+
+    def svg(self, nodecolor="blue", leafcolor="red", funccolor="green"):
+        """
+        :return: SVG representation of a tree.
+        """
+        fontsize = 12
+        hscale = 40
+        vscale = 25
+        hstart = vstart = 20
+        width = max(col for _, col in self.coords.values())
+        height = max(row for row, _ in self.coords.values())
+        result = [
+            '<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
+            'width="%dem" height="%dem" viewBox="%d %d %d %d">'
+            % (
+                width * 3,
+                height * 2.5,
+                -hstart,
+                -vstart,
+                width * hscale + 3 * hstart,
+                height * vscale + 3 * vstart,
+            )
+        ]
+
+        children = defaultdict(set)
+        for n in self.nodes:
+            if n:
+                children[self.edges[n]].add(n)
+
+        # horizontal branches from nodes to children
+        for node in self.nodes:
+            if not children[node]:
+                continue
+            y, x = self.coords[node]
+            x *= hscale
+            y *= vscale
+            x += hstart
+            y += vstart + fontsize // 2
+            childx = [self.coords[c][1] for c in children[node]]
+            xmin = hstart + hscale * min(childx)
+            xmax = hstart + hscale * max(childx)
+            result.append(
+                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
+                'points="%g,%g %g,%g" />' % (xmin, y, xmax, y)
+            )
+            result.append(
+                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
+                'points="%g,%g %g,%g" />' % (x, y, x, y - fontsize // 3)
+            )
+
+        # vertical branches from children to parents
+        for child, parent in self.edges.items():
+            y, _ = self.coords[parent]
+            y *= vscale
+            y += vstart + fontsize // 2
+            childy, childx = self.coords[child]
+            childx *= hscale
+            childy *= vscale
+            childx += hstart
+            childy += vstart - fontsize
+            result += [
+                '\t<polyline style="stroke:white; stroke-width:10; fill:none;"'
+                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y + 5),
+                '\t<polyline style="stroke:black; stroke-width:1; fill:none;"'
+                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y),
+            ]
+
+        # write nodes with coordinates
+        for n, (row, column) in self.coords.items():
+            node = self.nodes[n]
+            x = column * hscale + hstart
+            y = row * vscale + vstart
+            if n in self.highlight:
+                color = nodecolor if isinstance(node, Tree) else leafcolor
+                if isinstance(node, Tree) and node.label().startswith("-"):
+                    color = funccolor
+            else:
+                color = "black"
+            result += [
+                '\t<text style="text-anchor: middle; fill: %s; '
+                'font-size: %dpx;" x="%g" y="%g">%s</text>'
+                % (
+                    color,
+                    fontsize,
+                    x,
+                    y,
+                    escape(
+                        node.label() if isinstance(node, Tree) else node, quote=False
+                    ),
+                )
+            ]
+
+        result += ["</svg>"]
+        return "\n".join(result)
+
+
+def test():
+    """Do some tree drawing tests."""
+
+    def print_tree(n, tree, sentence=None, ansi=True, **xargs):
+        print()
+        print('{}: "{}"'.format(n, " ".join(sentence or tree.leaves())))
+        print(tree)
+        print()
+        drawtree = TreePrettyPrinter(tree, sentence)
+        try:
+            print(drawtree.text(unicodelines=ansi, ansi=ansi, **xargs))
+        except (UnicodeDecodeError, UnicodeEncodeError):
+            print(drawtree.text(unicodelines=False, ansi=False, **xargs))
+
+    from nltk.corpus import treebank
+
+    for n in [0, 1440, 1591, 2771, 2170]:
+        tree = treebank.parsed_sents()[n]
+        print_tree(n, tree, nodedist=2, maxwidth=8)
+    print()
+    print("ASCII version:")
+    print(TreePrettyPrinter(tree).text(nodedist=2))
+
+    tree = Tree.fromstring(
+        "(top (punct 8) (smain (noun 0) (verb 1) (inf (verb 5) (inf (verb 6) "
+        "(conj (inf (pp (prep 2) (np (det 3) (noun 4))) (verb 7)) (inf (verb 9)) "
+        "(vg 10) (inf (verb 11)))))) (punct 12))",
+        read_leaf=int,
+    )
+    sentence = (
+        "Ze had met haar moeder kunnen gaan winkelen ,"
+        " zwemmen of terrassen .".split()
+    )
+    print_tree("Discontinuous tree", tree, sentence, nodedist=2)
+
+
+__all__ = ["TreePrettyPrinter"]
+
+if __name__ == "__main__":
+    test()
diff --git a/nltk/tree/probabilistic.py b/nltk/tree/probabilistic.py
new file mode 100644
index 000000000..1d7384c80
--- /dev/null
+++ b/nltk/tree/probabilistic.py
@@ -0,0 +1,78 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+
+from nltk.internals import raise_unorderable_types
+from nltk.probability import ProbabilisticMixIn
+from nltk.tree.immutable import ImmutableProbabilisticTree
+from nltk.tree.tree import Tree
+
+######################################################################
+## Probabilistic trees
+######################################################################
+
+
+class ProbabilisticTree(Tree, ProbabilisticMixIn):
+    def __init__(self, node, children=None, **prob_kwargs):
+        Tree.__init__(self, node, children)
+        ProbabilisticMixIn.__init__(self, **prob_kwargs)
+
+    # We have to patch up these methods to make them work right:
+    def _frozen_class(self):
+        return ImmutableProbabilisticTree
+
+    def __repr__(self):
+        return f"{Tree.__repr__(self)} (p={self.prob()!r})"
+
+    def __str__(self):
+        return f"{self.pformat(margin=60)} (p={self.prob():.6g})"
+
+    def copy(self, deep=False):
+        if not deep:
+            return type(self)(self._label, self, prob=self.prob())
+        else:
+            return type(self).convert(self)
+
+    @classmethod
+    def convert(cls, val):
+        if isinstance(val, Tree):
+            children = [cls.convert(child) for child in val]
+            if isinstance(val, ProbabilisticMixIn):
+                return cls(val._label, children, prob=val.prob())
+            else:
+                return cls(val._label, children, prob=1.0)
+        else:
+            return val
+
+    def __eq__(self, other):
+        return (
+            self.__class__ is other.__class__
+            and (
+                self._label,
+                list(self),
+                self.prob(),
+            )
+            == (other._label, list(other), other.prob())
+        )
+
+    def __lt__(self, other):
+        if not isinstance(other, Tree):
+            raise_unorderable_types("<", self, other)
+        if self.__class__ is other.__class__:
+            return (self._label, list(self), self.prob()) < (
+                other._label,
+                list(other),
+                other.prob(),
+            )
+        else:
+            return self.__class__.__name__ < other.__class__.__name__
+
+
+__all__ = ["ProbabilisticTree"]
diff --git a/nltk/tree/transforms.py b/nltk/tree/transforms.py
new file mode 100644
index 000000000..2ebaa4dbf
--- /dev/null
+++ b/nltk/tree/transforms.py
@@ -0,0 +1,338 @@
+# Natural Language Toolkit: Tree Transformations
+#
+# Copyright (C) 2005-2007 Oregon Graduate Institute
+# Author: Nathan Bodenstab <bodenstab@cslu.ogi.edu>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+r"""
+A collection of methods for tree (grammar) transformations used
+in parsing natural language.
+
+Although many of these methods are technically grammar transformations
+(ie. Chomsky Norm Form), when working with treebanks it is much more
+natural to visualize these modifications in a tree structure.  Hence,
+we will do all transformation directly to the tree itself.
+Transforming the tree directly also allows us to do parent annotation.
+A grammar can then be simply induced from the modified tree.
+
+The following is a short tutorial on the available transformations.
+
+ 1. Chomsky Normal Form (binarization)
+
+    It is well known that any grammar has a Chomsky Normal Form (CNF)
+    equivalent grammar where CNF is defined by every production having
+    either two non-terminals or one terminal on its right hand side.
+    When we have hierarchically structured data (ie. a treebank), it is
+    natural to view this in terms of productions where the root of every
+    subtree is the head (left hand side) of the production and all of
+    its children are the right hand side constituents.  In order to
+    convert a tree into CNF, we simply need to ensure that every subtree
+    has either two subtrees as children (binarization), or one leaf node
+    (non-terminal).  In order to binarize a subtree with more than two
+    children, we must introduce artificial nodes.
+
+    There are two popular methods to convert a tree into CNF: left
+    factoring and right factoring.  The following example demonstrates
+    the difference between them.  Example::
+
+     Original       Right-Factored     Left-Factored
+
+          A              A                      A
+        / | \          /   \                  /   \
+       B  C  D   ==>  B    A|<C-D>   OR   A|<B-C>  D
+                            /  \          /  \
+                           C    D        B    C
+
+ 2. Parent Annotation
+
+    In addition to binarizing the tree, there are two standard
+    modifications to node labels we can do in the same traversal: parent
+    annotation and Markov order-N smoothing (or sibling smoothing).
+
+    The purpose of parent annotation is to refine the probabilities of
+    productions by adding a small amount of context.  With this simple
+    addition, a CYK (inside-outside, dynamic programming chart parse)
+    can improve from 74% to 79% accuracy.  A natural generalization from
+    parent annotation is to grandparent annotation and beyond.  The
+    tradeoff becomes accuracy gain vs. computational complexity.  We
+    must also keep in mind data sparcity issues.  Example::
+
+     Original       Parent Annotation
+
+          A                A^<?>
+        / | \             /   \
+       B  C  D   ==>  B^<A>    A|<C-D>^<?>     where ? is the
+                                 /  \          parent of A
+                             C^<A>   D^<A>
+
+
+ 3. Markov order-N smoothing
+
+    Markov smoothing combats data sparcity issues as well as decreasing
+    computational requirements by limiting the number of children
+    included in artificial nodes.  In practice, most people use an order
+    2 grammar.  Example::
+
+      Original       No Smoothing       Markov order 1   Markov order 2   etc.
+
+       __A__            A                      A                A
+      / /|\ \         /   \                  /   \            /   \
+     B C D E F  ==>  B    A|<C-D-E-F>  ==>  B   A|<C>  ==>   B  A|<C-D>
+                            /   \               /   \            /   \
+                           C    ...            C    ...         C    ...
+
+
+
+    Annotation decisions can be thought about in the vertical direction
+    (parent, grandparent, etc) and the horizontal direction (number of
+    siblings to keep).  Parameters to the following functions specify
+    these values.  For more information see:
+
+    Dan Klein and Chris Manning (2003) "Accurate Unlexicalized
+    Parsing", ACL-03.  https://www.aclweb.org/anthology/P03-1054
+
+ 4. Unary Collapsing
+
+    Collapse unary productions (ie. subtrees with a single child) into a
+    new non-terminal (Tree node).  This is useful when working with
+    algorithms that do not allow unary productions, yet you do not wish
+    to lose the parent information.  Example::
+
+       A
+       |
+       B   ==>   A+B
+      / \        / \
+     C   D      C   D
+
+"""
+
+from nltk.tree.tree import Tree
+
+
+def chomsky_normal_form(
+    tree, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^"
+):
+    # assume all subtrees have homogeneous children
+    # assume all terminals have no siblings
+
+    # A semi-hack to have elegant looking code below.  As a result,
+    # any subtree with a branching factor greater than 999 will be incorrectly truncated.
+    if horzMarkov is None:
+        horzMarkov = 999
+
+    # Traverse the tree depth-first keeping a list of ancestor nodes to the root.
+    # I chose not to use the tree.treepositions() method since it requires
+    # two traversals of the tree (one to get the positions, one to iterate
+    # over them) and node access time is proportional to the height of the node.
+    # This method is 7x faster which helps when parsing 40,000 sentences.
+
+    nodeList = [(tree, [tree.label()])]
+    while nodeList != []:
+        node, parent = nodeList.pop()
+        if isinstance(node, Tree):
+
+            # parent annotation
+            parentString = ""
+            originalNode = node.label()
+            if vertMarkov != 0 and node != tree and isinstance(node[0], Tree):
+                parentString = "{}<{}>".format(parentChar, "-".join(parent))
+                node.set_label(node.label() + parentString)
+                parent = [originalNode] + parent[: vertMarkov - 1]
+
+            # add children to the agenda before we mess with them
+            for child in node:
+                nodeList.append((child, parent))
+
+            # chomsky normal form factorization
+            if len(node) > 2:
+                childNodes = [child.label() for child in node]
+                nodeCopy = node.copy()
+                node[0:] = []  # delete the children
+
+                curNode = node
+                numChildren = len(nodeCopy)
+                for i in range(1, numChildren - 1):
+                    if factor == "right":
+                        newHead = "{}{}<{}>{}".format(
+                            originalNode,
+                            childChar,
+                            "-".join(
+                                childNodes[i : min([i + horzMarkov, numChildren])]
+                            ),
+                            parentString,
+                        )  # create new head
+                        newNode = Tree(newHead, [])
+                        curNode[0:] = [nodeCopy.pop(0), newNode]
+                    else:
+                        newHead = "{}{}<{}>{}".format(
+                            originalNode,
+                            childChar,
+                            "-".join(
+                                childNodes[max([numChildren - i - horzMarkov, 0]) : -i]
+                            ),
+                            parentString,
+                        )
+                        newNode = Tree(newHead, [])
+                        curNode[0:] = [newNode, nodeCopy.pop()]
+
+                    curNode = newNode
+
+                curNode[0:] = [child for child in nodeCopy]
+
+
+def un_chomsky_normal_form(
+    tree, expandUnary=True, childChar="|", parentChar="^", unaryChar="+"
+):
+    # Traverse the tree-depth first keeping a pointer to the parent for modification purposes.
+    nodeList = [(tree, [])]
+    while nodeList != []:
+        node, parent = nodeList.pop()
+        if isinstance(node, Tree):
+            # if the node contains the 'childChar' character it means that
+            # it is an artificial node and can be removed, although we still need
+            # to move its children to its parent
+            childIndex = node.label().find(childChar)
+            if childIndex != -1:
+                nodeIndex = parent.index(node)
+                parent.remove(parent[nodeIndex])
+                # Generated node was on the left if the nodeIndex is 0 which
+                # means the grammar was left factored.  We must insert the children
+                # at the beginning of the parent's children
+                if nodeIndex == 0:
+                    parent.insert(0, node[0])
+                    parent.insert(1, node[1])
+                else:
+                    parent.extend([node[0], node[1]])
+
+                # parent is now the current node so the children of parent will be added to the agenda
+                node = parent
+            else:
+                parentIndex = node.label().find(parentChar)
+                if parentIndex != -1:
+                    # strip the node name of the parent annotation
+                    node.set_label(node.label()[:parentIndex])
+
+                # expand collapsed unary productions
+                if expandUnary == True:
+                    unaryIndex = node.label().find(unaryChar)
+                    if unaryIndex != -1:
+                        newNode = Tree(
+                            node.label()[unaryIndex + 1 :], [i for i in node]
+                        )
+                        node.set_label(node.label()[:unaryIndex])
+                        node[0:] = [newNode]
+
+            for child in node:
+                nodeList.append((child, node))
+
+
+def collapse_unary(tree, collapsePOS=False, collapseRoot=False, joinChar="+"):
+    """
+    Collapse subtrees with a single child (ie. unary productions)
+    into a new non-terminal (Tree node) joined by 'joinChar'.
+    This is useful when working with algorithms that do not allow
+    unary productions, and completely removing the unary productions
+    would require loss of useful information.  The Tree is modified
+    directly (since it is passed by reference) and no value is returned.
+
+    :param tree: The Tree to be collapsed
+    :type  tree: Tree
+    :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie.
+                        Part-of-Speech tags) since they are always unary productions
+    :type  collapsePOS: bool
+    :param collapseRoot: 'False' (default) will not modify the root production
+                         if it is unary.  For the Penn WSJ treebank corpus, this corresponds
+                         to the TOP -> productions.
+    :type collapseRoot: bool
+    :param joinChar: A string used to connect collapsed node values (default = "+")
+    :type  joinChar: str
+    """
+
+    if collapseRoot == False and isinstance(tree, Tree) and len(tree) == 1:
+        nodeList = [tree[0]]
+    else:
+        nodeList = [tree]
+
+    # depth-first traversal of tree
+    while nodeList != []:
+        node = nodeList.pop()
+        if isinstance(node, Tree):
+            if (
+                len(node) == 1
+                and isinstance(node[0], Tree)
+                and (collapsePOS == True or isinstance(node[0, 0], Tree))
+            ):
+                node.set_label(node.label() + joinChar + node[0].label())
+                node[0:] = [child for child in node[0]]
+                # since we assigned the child's children to the current node,
+                # evaluate the current node again
+                nodeList.append(node)
+            else:
+                for child in node:
+                    nodeList.append(child)
+
+
+#################################################################
+# Demonstration
+#################################################################
+
+
+def demo():
+    """
+    A demonstration showing how each tree transform can be used.
+    """
+
+    from copy import deepcopy
+
+    from nltk.draw.tree import draw_trees
+    from nltk.tree.tree import Tree
+
+    # original tree from WSJ bracketed text
+    sentence = """(TOP
+  (S
+    (S
+      (VP
+        (VBN Turned)
+        (ADVP (RB loose))
+        (PP
+          (IN in)
+          (NP
+            (NP (NNP Shane) (NNP Longman) (POS 's))
+            (NN trading)
+            (NN room)))))
+    (, ,)
+    (NP (DT the) (NN yuppie) (NNS dealers))
+    (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
+    (. .)))"""
+    t = Tree.fromstring(sentence, remove_empty_top_bracketing=True)
+
+    # collapse subtrees with only one child
+    collapsedTree = deepcopy(t)
+    collapse_unary(collapsedTree)
+
+    # convert the tree to CNF
+    cnfTree = deepcopy(collapsedTree)
+    chomsky_normal_form(cnfTree)
+
+    # convert the tree to CNF with parent annotation (one level) and horizontal smoothing of order two
+    parentTree = deepcopy(collapsedTree)
+    chomsky_normal_form(parentTree, horzMarkov=2, vertMarkov=1)
+
+    # convert the tree back to its original form (used to make CYK results comparable)
+    original = deepcopy(parentTree)
+    un_chomsky_normal_form(original)
+
+    # convert tree back to bracketed text
+    sentence2 = original.pprint()
+    print(sentence)
+    print(sentence2)
+    print("Sentences the same? ", sentence == sentence2)
+
+    draw_trees(t, collapsedTree, cnfTree, parentTree, original)
+
+
+if __name__ == "__main__":
+    demo()
+
+__all__ = ["chomsky_normal_form", "un_chomsky_normal_form", "collapse_unary"]
diff --git a/nltk/tree.py b/nltk/tree/tree.py
similarity index 52%
rename from nltk/tree.py
rename to nltk/tree/tree.py
index c9acd7d8f..6c983c4c6 100644
--- a/nltk/tree.py
+++ b/nltk/tree/tree.py
@@ -16,14 +16,8 @@ syntax trees and morphological trees.
 
 import re
 import sys
-from abc import ABCMeta, abstractmethod
 
 from nltk.grammar import Nonterminal, Production
-from nltk.internals import raise_unorderable_types
-from nltk.probability import ProbabilisticMixIn
-from nltk.util import slice_bounds
-
-# TODO: add LabelledTree (can be used for dependency trees)
 
 ######################################################################
 ## Trees
@@ -362,7 +356,7 @@ class Tree(list):
         form P -> C1 C2 ... Cn.
 
             >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
-            >>> t.productions()
+            >>> t.productions() # doctest: +NORMALIZE_WHITESPACE
             [S -> NP VP, NP -> D N, D -> 'the', N -> 'dog', VP -> V NP, V -> 'chased',
             NP -> D N, D -> 'the', N -> 'cat']
 
@@ -479,7 +473,7 @@ class Tree(list):
         :param parentChar: A string used to separate the node representation from its vertical annotation
         :type  parentChar: str
         """
-        from nltk.treetransforms import chomsky_normal_form
+        from nltk.tree.transforms import chomsky_normal_form
 
         chomsky_normal_form(self, factor, horzMarkov, vertMarkov, childChar, parentChar)
 
@@ -504,7 +498,7 @@ class Tree(list):
         :param unaryChar: A string joining two non-terminals in a unary production (default = "+")
         :type  unaryChar: str
         """
-        from nltk.treetransforms import un_chomsky_normal_form
+        from nltk.tree.transforms import un_chomsky_normal_form
 
         un_chomsky_normal_form(self, expandUnary, childChar, parentChar, unaryChar)
 
@@ -527,7 +521,7 @@ class Tree(list):
         :param joinChar: A string used to connect collapsed node values (default = "+")
         :type  joinChar: str
         """
-        from nltk.treetransforms import collapse_unary
+        from nltk.tree.transforms import collapse_unary
 
         collapse_unary(self, collapsePOS, collapseRoot, joinChar)
 
@@ -564,6 +558,8 @@ class Tree(list):
             return type(self).convert(self)
 
     def _frozen_class(self):
+        from nltk.tree.immutable import ImmutableTree
+
         return ImmutableTree
 
     def freeze(self, leaf_freezer=None):
@@ -768,9 +764,9 @@ class Tree(list):
         """
         Pretty-print this tree as ASCII or Unicode art.
         For explanation of the arguments, see the documentation for
-        `nltk.treeprettyprinter.TreePrettyPrinter`.
+        `nltk.tree.prettyprinter.TreePrettyPrinter`.
         """
-        from nltk.treeprettyprinter import TreePrettyPrinter
+        from nltk.tree.prettyprinter import TreePrettyPrinter
 
         print(TreePrettyPrinter(self, sentence, highlight).text(**kwargs), file=stream)
 
@@ -782,61 +778,10 @@ class Tree(list):
             childstr,
         )
 
-    def _repr_png_(self):
-        """
-        Draws and outputs in PNG for ipython.
-        PNG is used instead of PDF, since it can be displayed in the qt console and
-        has wider browser support.
-        """
-        import base64
-        import os
-        import subprocess
-        import tempfile
-
-        from nltk.draw.tree import tree_to_treesegment
-        from nltk.draw.util import CanvasFrame
-        from nltk.internals import find_binary
-
-        _canvas_frame = CanvasFrame()
-        widget = tree_to_treesegment(_canvas_frame.canvas(), self)
-        _canvas_frame.add_widget(widget)
-        x, y, w, h = widget.bbox()
-        # print_to_file uses scrollregion to set the width and height of the pdf.
-        _canvas_frame.canvas()["scrollregion"] = (0, 0, w, h)
-        with tempfile.NamedTemporaryFile() as file:
-            in_path = f"{file.name}.ps"
-            out_path = f"{file.name}.png"
-            _canvas_frame.print_to_file(in_path)
-            _canvas_frame.destroy_widget(widget)
-            try:
-                subprocess.call(
-                    [
-                        find_binary(
-                            "gs",
-                            binary_names=["gswin32c.exe", "gswin64c.exe"],
-                            env_vars=["PATH"],
-                            verbose=False,
-                        )
-                    ]
-                    + "-q -dEPSCrop -sDEVICE=png16m -r90 -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -dSAFER -dBATCH -dNOPAUSE -sOutputFile={} {}".format(
-                        out_path, in_path
-                    ).split()
-                )
-            except LookupError as e:
-                pre_error_message = str(
-                    "The Ghostscript executable isn't found.\n"
-                    "See https://web.mit.edu/ghostscript/www/Install.htm\n"
-                    "If you're using a Mac, you can try installing\n"
-                    "https://docs.brew.sh/Installation then `brew install ghostscript`"
-                )
-                print(pre_error_message, file=sys.stderr)
-                raise LookupError from e
+    def _repr_svg_(self):
+        from svgling import draw_tree
 
-            with open(out_path, "rb") as sr:
-                res = sr.read()
-            os.remove(in_path)
-            os.remove(out_path)
-            return base64.b64encode(res).decode()
+        return draw_tree(self)._repr_svg_()
 
     def __str__(self):
         return self.pformat()
@@ -945,718 +890,6 @@ class Tree(list):
             )
 
 
-class ImmutableTree(Tree):
-    def __init__(self, node, children=None):
-        super().__init__(node, children)
-        # Precompute our hash value.  This ensures that we're really
-        # immutable.  It also means we only have to calculate it once.
-        try:
-            self._hash = hash((self._label, tuple(self)))
-        except (TypeError, ValueError) as e:
-            raise ValueError(
-                "%s: node value and children " "must be immutable" % type(self).__name__
-            ) from e
-
-    def __setitem__(self, index, value):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __setslice__(self, i, j, value):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __delitem__(self, index):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __delslice__(self, i, j):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __iadd__(self, other):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __imul__(self, other):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def append(self, v):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def extend(self, v):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def pop(self, v=None):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def remove(self, v):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def reverse(self):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def sort(self):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __hash__(self):
-        return self._hash
-
-    def set_label(self, value):
-        """
-        Set the node label.  This will only succeed the first time the
-        node label is set, which should occur in ImmutableTree.__init__().
-        """
-        if hasattr(self, "_label"):
-            raise ValueError("%s may not be modified" % type(self).__name__)
-        self._label = value
-
-
-######################################################################
-## Parented trees
-######################################################################
-class AbstractParentedTree(Tree, metaclass=ABCMeta):
-    """
-    An abstract base class for a ``Tree`` that automatically maintains
-    pointers to parent nodes.  These parent pointers are updated
-    whenever any change is made to a tree's structure.  Two subclasses
-    are currently defined:
-
-      - ``ParentedTree`` is used for tree structures where each subtree
-        has at most one parent.  This class should be used in cases
-        where there is no"sharing" of subtrees.
-
-      - ``MultiParentedTree`` is used for tree structures where a
-        subtree may have zero or more parents.  This class should be
-        used in cases where subtrees may be shared.
-
-    Subclassing
-    ===========
-    The ``AbstractParentedTree`` class redefines all operations that
-    modify a tree's structure to call two methods, which are used by
-    subclasses to update parent information:
-
-      - ``_setparent()`` is called whenever a new child is added.
-      - ``_delparent()`` is called whenever a child is removed.
-    """
-
-    def __init__(self, node, children=None):
-        super().__init__(node, children)
-        # If children is None, the tree is read from node, and
-        # all parents will be set during parsing.
-        if children is not None:
-            # Otherwise we have to set the parent of the children.
-            # Iterate over self, and *not* children, because children
-            # might be an iterator.
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    self._setparent(child, i, dry_run=True)
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    self._setparent(child, i)
-
-    # ////////////////////////////////////////////////////////////
-    # Parent management
-    # ////////////////////////////////////////////////////////////
-    @abstractmethod
-    def _setparent(self, child, index, dry_run=False):
-        """
-        Update the parent pointer of ``child`` to point to ``self``.  This
-        method is only called if the type of ``child`` is ``Tree``;
-        i.e., it is not called when adding a leaf to a tree.  This method
-        is always called before the child is actually added to the
-        child list of ``self``.
-
-        :type child: Tree
-        :type index: int
-        :param index: The index of ``child`` in ``self``.
-        :raise TypeError: If ``child`` is a tree with an impropriate
-            type.  Typically, if ``child`` is a tree, then its type needs
-            to match the type of ``self``.  This prevents mixing of
-            different tree types (single-parented, multi-parented, and
-            non-parented).
-        :param dry_run: If true, the don't actually set the child's
-            parent pointer; just check for any error conditions, and
-            raise an exception if one is found.
-        """
-
-    @abstractmethod
-    def _delparent(self, child, index):
-        """
-        Update the parent pointer of ``child`` to not point to self.  This
-        method is only called if the type of ``child`` is ``Tree``; i.e., it
-        is not called when removing a leaf from a tree.  This method
-        is always called before the child is actually removed from the
-        child list of ``self``.
-
-        :type child: Tree
-        :type index: int
-        :param index: The index of ``child`` in ``self``.
-        """
-
-    # ////////////////////////////////////////////////////////////
-    # Methods that add/remove children
-    # ////////////////////////////////////////////////////////////
-    # Every method that adds or removes a child must make
-    # appropriate calls to _setparent() and _delparent().
-
-    def __delitem__(self, index):
-        # del ptree[start:stop]
-        if isinstance(index, slice):
-            start, stop, step = slice_bounds(self, index, allow_step=True)
-            # Clear all the children pointers.
-            for i in range(start, stop, step):
-                if isinstance(self[i], Tree):
-                    self._delparent(self[i], i)
-            # Delete the children from our child list.
-            super().__delitem__(index)
-
-        # del ptree[i]
-        elif isinstance(index, int):
-            if index < 0:
-                index += len(self)
-            if index < 0:
-                raise IndexError("index out of range")
-            # Clear the child's parent pointer.
-            if isinstance(self[index], Tree):
-                self._delparent(self[index], index)
-            # Remove the child from our child list.
-            super().__delitem__(index)
-
-        elif isinstance(index, (list, tuple)):
-            # del ptree[()]
-            if len(index) == 0:
-                raise IndexError("The tree position () may not be deleted.")
-            # del ptree[(i,)]
-            elif len(index) == 1:
-                del self[index[0]]
-            # del ptree[i1, i2, i3]
-            else:
-                del self[index[0]][index[1:]]
-
-        else:
-            raise TypeError(
-                "%s indices must be integers, not %s"
-                % (type(self).__name__, type(index).__name__)
-            )
-
-    def __setitem__(self, index, value):
-        # ptree[start:stop] = value
-        if isinstance(index, slice):
-            start, stop, step = slice_bounds(self, index, allow_step=True)
-            # make a copy of value, in case it's an iterator
-            if not isinstance(value, (list, tuple)):
-                value = list(value)
-            # Check for any error conditions, so we can avoid ending
-            # up in an inconsistent state if an error does occur.
-            for i, child in enumerate(value):
-                if isinstance(child, Tree):
-                    self._setparent(child, start + i * step, dry_run=True)
-            # clear the child pointers of all parents we're removing
-            for i in range(start, stop, step):
-                if isinstance(self[i], Tree):
-                    self._delparent(self[i], i)
-            # set the child pointers of the new children.  We do this
-            # after clearing *all* child pointers, in case we're e.g.
-            # reversing the elements in a tree.
-            for i, child in enumerate(value):
-                if isinstance(child, Tree):
-                    self._setparent(child, start + i * step)
-            # finally, update the content of the child list itself.
-            super().__setitem__(index, value)
-
-        # ptree[i] = value
-        elif isinstance(index, int):
-            if index < 0:
-                index += len(self)
-            if index < 0:
-                raise IndexError("index out of range")
-            # if the value is not changing, do nothing.
-            if value is self[index]:
-                return
-            # Set the new child's parent pointer.
-            if isinstance(value, Tree):
-                self._setparent(value, index)
-            # Remove the old child's parent pointer
-            if isinstance(self[index], Tree):
-                self._delparent(self[index], index)
-            # Update our child list.
-            super().__setitem__(index, value)
-
-        elif isinstance(index, (list, tuple)):
-            # ptree[()] = value
-            if len(index) == 0:
-                raise IndexError("The tree position () may not be assigned to.")
-            # ptree[(i,)] = value
-            elif len(index) == 1:
-                self[index[0]] = value
-            # ptree[i1, i2, i3] = value
-            else:
-                self[index[0]][index[1:]] = value
-
-        else:
-            raise TypeError(
-                "%s indices must be integers, not %s"
-                % (type(self).__name__, type(index).__name__)
-            )
-
-    def append(self, child):
-        if isinstance(child, Tree):
-            self._setparent(child, len(self))
-        super().append(child)
-
-    def extend(self, children):
-        for child in children:
-            if isinstance(child, Tree):
-                self._setparent(child, len(self))
-            super().append(child)
-
-    def insert(self, index, child):
-        # Handle negative indexes.  Note that if index < -len(self),
-        # we do *not* raise an IndexError, unlike __getitem__.  This
-        # is done for consistency with list.__getitem__ and list.index.
-        if index < 0:
-            index += len(self)
-        if index < 0:
-            index = 0
-        # Set the child's parent, and update our child list.
-        if isinstance(child, Tree):
-            self._setparent(child, index)
-        super().insert(index, child)
-
-    def pop(self, index=-1):
-        if index < 0:
-            index += len(self)
-        if index < 0:
-            raise IndexError("index out of range")
-        if isinstance(self[index], Tree):
-            self._delparent(self[index], index)
-        return super().pop(index)
-
-    # n.b.: like `list`, this is done by equality, not identity!
-    # To remove a specific child, use del ptree[i].
-    def remove(self, child):
-        index = self.index(child)
-        if isinstance(self[index], Tree):
-            self._delparent(self[index], index)
-        super().remove(child)
-
-    # We need to implement __getslice__ and friends, even though
-    # they're deprecated, because otherwise list.__getslice__ will get
-    # called (since we're subclassing from list).  Just delegate to
-    # __getitem__ etc., but use max(0, start) and max(0, stop) because
-    # because negative indices are already handled *before*
-    # __getslice__ is called; and we don't want to double-count them.
-    if hasattr(list, "__getslice__"):
-
-        def __getslice__(self, start, stop):
-            return self.__getitem__(slice(max(0, start), max(0, stop)))
-
-        def __delslice__(self, start, stop):
-            return self.__delitem__(slice(max(0, start), max(0, stop)))
-
-        def __setslice__(self, start, stop, value):
-            return self.__setitem__(slice(max(0, start), max(0, stop)), value)
-
-
-class ParentedTree(AbstractParentedTree):
-    """
-    A ``Tree`` that automatically maintains parent pointers for
-    single-parented trees.  The following are methods for querying
-    the structure of a parented tree: ``parent``, ``parent_index``,
-    ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``.
-
-    Each ``ParentedTree`` may have at most one parent.  In
-    particular, subtrees may not be shared.  Any attempt to reuse a
-    single ``ParentedTree`` as a child of more than one parent (or
-    as multiple children of the same parent) will cause a
-    ``ValueError`` exception to be raised.
-
-    ``ParentedTrees`` should never be used in the same tree as ``Trees``
-    or ``MultiParentedTrees``.  Mixing tree implementations may result
-    in incorrect parent pointers and in ``TypeError`` exceptions.
-    """
-
-    def __init__(self, node, children=None):
-        self._parent = None
-        """The parent of this Tree, or None if it has no parent."""
-        super().__init__(node, children)
-        if children is None:
-            # If children is None, the tree is read from node.
-            # After parsing, the parent of the immediate children
-            # will point to an intermediate tree, not self.
-            # We fix this by brute force:
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    child._parent = None
-                    self._setparent(child, i)
-
-    def _frozen_class(self):
-        return ImmutableParentedTree
-
-    # /////////////////////////////////////////////////////////////////
-    # Methods
-    # /////////////////////////////////////////////////////////////////
-
-    def parent(self):
-        """The parent of this tree, or None if it has no parent."""
-        return self._parent
-
-    def parent_index(self):
-        """
-        The index of this tree in its parent.  I.e.,
-        ``ptree.parent()[ptree.parent_index()] is ptree``.  Note that
-        ``ptree.parent_index()`` is not necessarily equal to
-        ``ptree.parent.index(ptree)``, since the ``index()`` method
-        returns the first child that is equal to its argument.
-        """
-        if self._parent is None:
-            return None
-        for i, child in enumerate(self._parent):
-            if child is self:
-                return i
-        assert False, "expected to find self in self._parent!"
-
-    def left_sibling(self):
-        """The left sibling of this tree, or None if it has none."""
-        parent_index = self.parent_index()
-        if self._parent and parent_index > 0:
-            return self._parent[parent_index - 1]
-        return None  # no left sibling
-
-    def right_sibling(self):
-        """The right sibling of this tree, or None if it has none."""
-        parent_index = self.parent_index()
-        if self._parent and parent_index < (len(self._parent) - 1):
-            return self._parent[parent_index + 1]
-        return None  # no right sibling
-
-    def root(self):
-        """
-        The root of this tree.  I.e., the unique ancestor of this tree
-        whose parent is None.  If ``ptree.parent()`` is None, then
-        ``ptree`` is its own root.
-        """
-        root = self
-        while root.parent() is not None:
-            root = root.parent()
-        return root
-
-    def treeposition(self):
-        """
-        The tree position of this tree, relative to the root of the
-        tree.  I.e., ``ptree.root[ptree.treeposition] is ptree``.
-        """
-        if self.parent() is None:
-            return ()
-        else:
-            return self.parent().treeposition() + (self.parent_index(),)
-
-    # /////////////////////////////////////////////////////////////////
-    # Parent Management
-    # /////////////////////////////////////////////////////////////////
-
-    def _delparent(self, child, index):
-        # Sanity checks
-        assert isinstance(child, ParentedTree)
-        assert self[index] is child
-        assert child._parent is self
-
-        # Delete child's parent pointer.
-        child._parent = None
-
-    def _setparent(self, child, index, dry_run=False):
-        # If the child's type is incorrect, then complain.
-        if not isinstance(child, ParentedTree):
-            raise TypeError(
-                "Can not insert a non-ParentedTree " + "into a ParentedTree"
-            )
-
-        # If child already has a parent, then complain.
-        if child._parent is not None:
-            raise ValueError("Can not insert a subtree that already " "has a parent.")
-
-        # Set child's parent pointer & index.
-        if not dry_run:
-            child._parent = self
-
-
-class MultiParentedTree(AbstractParentedTree):
-    """
-    A ``Tree`` that automatically maintains parent pointers for
-    multi-parented trees.  The following are methods for querying the
-    structure of a multi-parented tree: ``parents()``, ``parent_indices()``,
-    ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``.
-
-    Each ``MultiParentedTree`` may have zero or more parents.  In
-    particular, subtrees may be shared.  If a single
-    ``MultiParentedTree`` is used as multiple children of the same
-    parent, then that parent will appear multiple times in its
-    ``parents()`` method.
-
-    ``MultiParentedTrees`` should never be used in the same tree as
-    ``Trees`` or ``ParentedTrees``.  Mixing tree implementations may
-    result in incorrect parent pointers and in ``TypeError`` exceptions.
-    """
-
-    def __init__(self, node, children=None):
-        self._parents = []
-        """A list of this tree's parents.  This list should not
-           contain duplicates, even if a parent contains this tree
-           multiple times."""
-        super().__init__(node, children)
-        if children is None:
-            # If children is None, the tree is read from node.
-            # After parsing, the parent(s) of the immediate children
-            # will point to an intermediate tree, not self.
-            # We fix this by brute force:
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    child._parents = []
-                    self._setparent(child, i)
-
-    def _frozen_class(self):
-        return ImmutableMultiParentedTree
-
-    # /////////////////////////////////////////////////////////////////
-    # Methods
-    # /////////////////////////////////////////////////////////////////
-
-    def parents(self):
-        """
-        The set of parents of this tree.  If this tree has no parents,
-        then ``parents`` is the empty set.  To check if a tree is used
-        as multiple children of the same parent, use the
-        ``parent_indices()`` method.
-
-        :type: list(MultiParentedTree)
-        """
-        return list(self._parents)
-
-    def left_siblings(self):
-        """
-        A list of all left siblings of this tree, in any of its parent
-        trees.  A tree may be its own left sibling if it is used as
-        multiple contiguous children of the same parent.  A tree may
-        appear multiple times in this list if it is the left sibling
-        of this tree with respect to multiple parents.
-
-        :type: list(MultiParentedTree)
-        """
-        return [
-            parent[index - 1]
-            for (parent, index) in self._get_parent_indices()
-            if index > 0
-        ]
-
-    def right_siblings(self):
-        """
-        A list of all right siblings of this tree, in any of its parent
-        trees.  A tree may be its own right sibling if it is used as
-        multiple contiguous children of the same parent.  A tree may
-        appear multiple times in this list if it is the right sibling
-        of this tree with respect to multiple parents.
-
-        :type: list(MultiParentedTree)
-        """
-        return [
-            parent[index + 1]
-            for (parent, index) in self._get_parent_indices()
-            if index < (len(parent) - 1)
-        ]
-
-    def _get_parent_indices(self):
-        return [
-            (parent, index)
-            for parent in self._parents
-            for index, child in enumerate(parent)
-            if child is self
-        ]
-
-    def roots(self):
-        """
-        The set of all roots of this tree.  This set is formed by
-        tracing all possible parent paths until trees with no parents
-        are found.
-
-        :type: list(MultiParentedTree)
-        """
-        return list(self._get_roots_helper({}).values())
-
-    def _get_roots_helper(self, result):
-        if self._parents:
-            for parent in self._parents:
-                parent._get_roots_helper(result)
-        else:
-            result[id(self)] = self
-        return result
-
-    def parent_indices(self, parent):
-        """
-        Return a list of the indices where this tree occurs as a child
-        of ``parent``.  If this child does not occur as a child of
-        ``parent``, then the empty list is returned.  The following is
-        always true::
-
-          for parent_index in ptree.parent_indices(parent):
-              parent[parent_index] is ptree
-        """
-        if parent not in self._parents:
-            return []
-        else:
-            return [index for (index, child) in enumerate(parent) if child is self]
-
-    def treepositions(self, root):
-        """
-        Return a list of all tree positions that can be used to reach
-        this multi-parented tree starting from ``root``.  I.e., the
-        following is always true::
-
-          for treepos in ptree.treepositions(root):
-              root[treepos] is ptree
-        """
-        if self is root:
-            return [()]
-        else:
-            return [
-                treepos + (index,)
-                for parent in self._parents
-                for treepos in parent.treepositions(root)
-                for (index, child) in enumerate(parent)
-                if child is self
-            ]
-
-    # /////////////////////////////////////////////////////////////////
-    # Parent Management
-    # /////////////////////////////////////////////////////////////////
-
-    def _delparent(self, child, index):
-        # Sanity checks
-        assert isinstance(child, MultiParentedTree)
-        assert self[index] is child
-        assert len([p for p in child._parents if p is self]) == 1
-
-        # If the only copy of child in self is at index, then delete
-        # self from child's parent list.
-        for i, c in enumerate(self):
-            if c is child and i != index:
-                break
-        else:
-            child._parents.remove(self)
-
-    def _setparent(self, child, index, dry_run=False):
-        # If the child's type is incorrect, then complain.
-        if not isinstance(child, MultiParentedTree):
-            raise TypeError(
-                "Can not insert a non-MultiParentedTree " + "into a MultiParentedTree"
-            )
-
-        # Add self as a parent pointer if it's not already listed.
-        if not dry_run:
-            for parent in child._parents:
-                if parent is self:
-                    break
-            else:
-                child._parents.append(self)
-
-
-class ImmutableParentedTree(ImmutableTree, ParentedTree):
-    pass
-
-
-class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree):
-    pass
-
-
-######################################################################
-## Probabilistic trees
-######################################################################
-
-
-class ProbabilisticTree(Tree, ProbabilisticMixIn):
-    def __init__(self, node, children=None, **prob_kwargs):
-        Tree.__init__(self, node, children)
-        ProbabilisticMixIn.__init__(self, **prob_kwargs)
-
-    # We have to patch up these methods to make them work right:
-    def _frozen_class(self):
-        return ImmutableProbabilisticTree
-
-    def __repr__(self):
-        return f"{Tree.__repr__(self)} (p={self.prob()!r})"
-
-    def __str__(self):
-        return f"{self.pformat(margin=60)} (p={self.prob():.6g})"
-
-    def copy(self, deep=False):
-        if not deep:
-            return type(self)(self._label, self, prob=self.prob())
-        else:
-            return type(self).convert(self)
-
-    @classmethod
-    def convert(cls, val):
-        if isinstance(val, Tree):
-            children = [cls.convert(child) for child in val]
-            if isinstance(val, ProbabilisticMixIn):
-                return cls(val._label, children, prob=val.prob())
-            else:
-                return cls(val._label, children, prob=1.0)
-        else:
-            return val
-
-    def __eq__(self, other):
-        return (
-            self.__class__ is other.__class__
-            and (
-                self._label,
-                list(self),
-                self.prob(),
-            )
-            == (other._label, list(other), other.prob())
-        )
-
-    def __lt__(self, other):
-        if not isinstance(other, Tree):
-            raise_unorderable_types("<", self, other)
-        if self.__class__ is other.__class__:
-            return (self._label, list(self), self.prob()) < (
-                other._label,
-                list(other),
-                other.prob(),
-            )
-        else:
-            return self.__class__.__name__ < other.__class__.__name__
-
-
-class ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn):
-    def __init__(self, node, children=None, **prob_kwargs):
-        ImmutableTree.__init__(self, node, children)
-        ProbabilisticMixIn.__init__(self, **prob_kwargs)
-        self._hash = hash((self._label, tuple(self), self.prob()))
-
-    # We have to patch up these methods to make them work right:
-    def _frozen_class(self):
-        return ImmutableProbabilisticTree
-
-    def __repr__(self):
-        return f"{Tree.__repr__(self)} [{self.prob()}]"
-
-    def __str__(self):
-        return f"{self.pformat(margin=60)} [{self.prob()}]"
-
-    def copy(self, deep=False):
-        if not deep:
-            return type(self)(self._label, self, prob=self.prob())
-        else:
-            return type(self).convert(self)
-
-    @classmethod
-    def convert(cls, val):
-        if isinstance(val, Tree):
-            children = [cls.convert(child) for child in val]
-            if isinstance(val, ProbabilisticMixIn):
-                return cls(val._label, children, prob=val.prob())
-            else:
-                return cls(val._label, children, prob=1.0)
-        else:
-            return val
-
-
 def _child_names(tree):
     names = []
     for child in tree:
@@ -1667,55 +900,6 @@ def _child_names(tree):
     return names
 
 
-######################################################################
-## Parsing
-######################################################################
-
-
-def bracket_parse(s):
-    """
-    Use Tree.read(s, remove_empty_top_bracketing=True) instead.
-    """
-    raise NameError("Use Tree.read(s, remove_empty_top_bracketing=True) instead.")
-
-
-def sinica_parse(s):
-    """
-    Parse a Sinica Treebank string and return a tree.  Trees are represented as nested brackettings,
-    as shown in the following example (X represents a Chinese character):
-    S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY)
-
-    :return: A tree corresponding to the string representation.
-    :rtype: Tree
-    :param s: The string to be converted
-    :type s: str
-    """
-    tokens = re.split(r"([()| ])", s)
-    for i in range(len(tokens)):
-        if tokens[i] == "(":
-            tokens[i - 1], tokens[i] = (
-                tokens[i],
-                tokens[i - 1],
-            )  # pull nonterminal inside parens
-        elif ":" in tokens[i]:
-            fields = tokens[i].split(":")
-            if len(fields) == 2:  # non-terminal
-                tokens[i] = fields[1]
-            else:
-                tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")"
-        elif tokens[i] == "|":
-            tokens[i] = ""
-
-    treebank_string = " ".join(tokens)
-    return Tree.fromstring(treebank_string, remove_empty_top_bracketing=True)
-
-
-#    s = re.sub(r'^#[^\s]*\s', '', s)  # remove leading identifier
-#    s = re.sub(r'\w+:', '', s)       # remove role tags
-
-#    return s
-
-
 ######################################################################
 ## Demonstration
 ######################################################################
@@ -1794,15 +978,5 @@ def demo():
 
 
 __all__ = [
-    "ImmutableProbabilisticTree",
-    "ImmutableTree",
-    "ProbabilisticMixIn",
-    "ProbabilisticTree",
     "Tree",
-    "bracket_parse",
-    "sinica_parse",
-    "ParentedTree",
-    "MultiParentedTree",
-    "ImmutableParentedTree",
-    "ImmutableMultiParentedTree",
 ]
diff --git a/nltk/treeprettyprinter.py b/nltk/treeprettyprinter.py
index defdf5e9d..22686d75a 100644
--- a/nltk/treeprettyprinter.py
+++ b/nltk/treeprettyprinter.py
@@ -17,612 +17,12 @@ Graph Algorithms and Applications, 10(2) 141--157 (2006)149.
 https://jgaa.info/accepted/2006/EschbachGuentherBecker2006.10.2.pdf
 """
 
-import re
+from nltk.internals import Deprecated
+from nltk.tree.prettyprinter import TreePrettyPrinter as TPP
 
-try:
-    from html import escape
-except ImportError:
-    from cgi import escape
 
-from collections import defaultdict
-from operator import itemgetter
-
-from nltk.tree import Tree
-from nltk.util import OrderedDict
-
-ANSICOLOR = {
-    "black": 30,
-    "red": 31,
-    "green": 32,
-    "yellow": 33,
-    "blue": 34,
-    "magenta": 35,
-    "cyan": 36,
-    "white": 37,
-}
-
-
-class TreePrettyPrinter:
-    """
-    Pretty-print a tree in text format, either as ASCII or Unicode.
-    The tree can be a normal tree, or discontinuous.
-
-    ``TreePrettyPrinter(tree, sentence=None, highlight=())``
-    creates an object from which different visualizations can be created.
-
-    :param tree: a Tree object.
-    :param sentence: a list of words (strings). If `sentence` is given,
-        `tree` must contain integers as leaves, which are taken as indices
-        in `sentence`. Using this you can display a discontinuous tree.
-    :param highlight: Optionally, a sequence of Tree objects in `tree` which
-        should be highlighted. Has the effect of only applying colors to nodes
-        in this sequence (nodes should be given as Tree objects, terminals as
-        indices).
-
-    >>> from nltk.tree import Tree
-    >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))')
-    >>> print(TreePrettyPrinter(tree).text())
-    ... # doctest: +NORMALIZE_WHITESPACE
-          S
-      ____|____
-     NP        VP
-     |         |
-    Mary     walks
-    """
-
-    def __init__(self, tree, sentence=None, highlight=()):
-        if sentence is None:
-            leaves = tree.leaves()
-            if (
-                leaves
-                and not any(len(a) == 0 for a in tree.subtrees())
-                and all(isinstance(a, int) for a in leaves)
-            ):
-                sentence = [str(a) for a in leaves]
-            else:
-                # this deals with empty nodes (frontier non-terminals)
-                # and multiple/mixed terminals under non-terminals.
-                tree = tree.copy(True)
-                sentence = []
-                for a in tree.subtrees():
-                    if len(a) == 0:
-                        a.append(len(sentence))
-                        sentence.append(None)
-                    elif any(not isinstance(b, Tree) for b in a):
-                        for n, b in enumerate(a):
-                            if not isinstance(b, Tree):
-                                a[n] = len(sentence)
-                                if type(b) == tuple:
-                                    b = "/".join(b)
-                                sentence.append("%s" % b)
-        self.nodes, self.coords, self.edges, self.highlight = self.nodecoords(
-            tree, sentence, highlight
-        )
-
-    def __str__(self):
-        return self.text()
-
-    def __repr__(self):
-        return "<TreePrettyPrinter with %d nodes>" % len(self.nodes)
-
-    @staticmethod
-    def nodecoords(tree, sentence, highlight):
-        """
-        Produce coordinates of nodes on a grid.
-
-        Objective:
-
-        - Produce coordinates for a non-overlapping placement of nodes and
-            horizontal lines.
-        - Order edges so that crossing edges cross a minimal number of previous
-            horizontal lines (never vertical lines).
-
-        Approach:
-
-        - bottom up level order traversal (start at terminals)
-        - at each level, identify nodes which cannot be on the same row
-        - identify nodes which cannot be in the same column
-        - place nodes into a grid at (row, column)
-        - order child-parent edges with crossing edges last
-
-        Coordinates are (row, column); the origin (0, 0) is at the top left;
-        the root node is on row 0. Coordinates do not consider the size of a
-        node (which depends on font, &c), so the width of a column of the grid
-        should be automatically determined by the element with the greatest
-        width in that column. Alternatively, the integer coordinates could be
-        converted to coordinates in which the distances between adjacent nodes
-        are non-uniform.
-
-        Produces tuple (nodes, coords, edges, highlighted) where:
-
-        - nodes[id]: Tree object for the node with this integer id
-        - coords[id]: (n, m) coordinate where to draw node with id in the grid
-        - edges[id]: parent id of node with this id (ordered dictionary)
-        - highlighted: set of ids that should be highlighted
-        """
-
-        def findcell(m, matrix, startoflevel, children):
-            """
-            Find vacant row, column index for node ``m``.
-            Iterate over current rows for this level (try lowest first)
-            and look for cell between first and last child of this node,
-            add new row to level if no free row available.
-            """
-            candidates = [a for _, a in children[m]]
-            minidx, maxidx = min(candidates), max(candidates)
-            leaves = tree[m].leaves()
-            center = scale * sum(leaves) // len(leaves)  # center of gravity
-            if minidx < maxidx and not minidx < center < maxidx:
-                center = sum(candidates) // len(candidates)
-            if max(candidates) - min(candidates) > 2 * scale:
-                center -= center % scale  # round to unscaled coordinate
-                if minidx < maxidx and not minidx < center < maxidx:
-                    center += scale
-            if ids[m] == 0:
-                startoflevel = len(matrix)
-            for rowidx in range(startoflevel, len(matrix) + 1):
-                if rowidx == len(matrix):  # need to add a new row
-                    matrix.append(
-                        [
-                            vertline if a not in (corner, None) else None
-                            for a in matrix[-1]
-                        ]
-                    )
-                row = matrix[rowidx]
-                i = j = center
-                if len(children[m]) == 1:  # place unaries directly above child
-                    return rowidx, next(iter(children[m]))[1]
-                elif all(
-                    a is None or a == vertline
-                    for a in row[min(candidates) : max(candidates) + 1]
-                ):
-                    # find free column
-                    for n in range(scale):
-                        i = j = center + n
-                        while j > minidx or i < maxidx:
-                            if i < maxidx and (
-                                matrix[rowidx][i] is None or i in candidates
-                            ):
-                                return rowidx, i
-                            elif j > minidx and (
-                                matrix[rowidx][j] is None or j in candidates
-                            ):
-                                return rowidx, j
-                            i += scale
-                            j -= scale
-            raise ValueError(
-                "could not find a free cell for:\n%s\n%s"
-                "min=%d; max=%d" % (tree[m], minidx, maxidx, dumpmatrix())
-            )
-
-        def dumpmatrix():
-            """Dump matrix contents for debugging purposes."""
-            return "\n".join(
-                "%2d: %s" % (n, " ".join(("%2r" % i)[:2] for i in row))
-                for n, row in enumerate(matrix)
-            )
-
-        leaves = tree.leaves()
-        if not all(isinstance(n, int) for n in leaves):
-            raise ValueError("All leaves must be integer indices.")
-        if len(leaves) != len(set(leaves)):
-            raise ValueError("Indices must occur at most once.")
-        if not all(0 <= n < len(sentence) for n in leaves):
-            raise ValueError(
-                "All leaves must be in the interval 0..n "
-                "with n=len(sentence)\ntokens: %d indices: "
-                "%r\nsentence: %s" % (len(sentence), tree.leaves(), sentence)
-            )
-        vertline, corner = -1, -2  # constants
-        tree = tree.copy(True)
-        for a in tree.subtrees():
-            a.sort(key=lambda n: min(n.leaves()) if isinstance(n, Tree) else n)
-        scale = 2
-        crossed = set()
-        # internal nodes and lexical nodes (no frontiers)
-        positions = tree.treepositions()
-        maxdepth = max(map(len, positions)) + 1
-        childcols = defaultdict(set)
-        matrix = [[None] * (len(sentence) * scale)]
-        nodes = {}
-        ids = {a: n for n, a in enumerate(positions)}
-        highlighted_nodes = {
-            n for a, n in ids.items() if not highlight or tree[a] in highlight
-        }
-        levels = {n: [] for n in range(maxdepth - 1)}
-        terminals = []
-        for a in positions:
-            node = tree[a]
-            if isinstance(node, Tree):
-                levels[maxdepth - node.height()].append(a)
-            else:
-                terminals.append(a)
-
-        for n in levels:
-            levels[n].sort(key=lambda n: max(tree[n].leaves()) - min(tree[n].leaves()))
-        terminals.sort()
-        positions = set(positions)
-
-        for m in terminals:
-            i = int(tree[m]) * scale
-            assert matrix[0][i] is None, (matrix[0][i], m, i)
-            matrix[0][i] = ids[m]
-            nodes[ids[m]] = sentence[tree[m]]
-            if nodes[ids[m]] is None:
-                nodes[ids[m]] = "..."
-                highlighted_nodes.discard(ids[m])
-            positions.remove(m)
-            childcols[m[:-1]].add((0, i))
-
-        # add other nodes centered on their children,
-        # if the center is already taken, back off
-        # to the left and right alternately, until an empty cell is found.
-        for n in sorted(levels, reverse=True):
-            nodesatdepth = levels[n]
-            startoflevel = len(matrix)
-            matrix.append(
-                [vertline if a not in (corner, None) else None for a in matrix[-1]]
-            )
-            for m in nodesatdepth:  # [::-1]:
-                if n < maxdepth - 1 and childcols[m]:
-                    _, pivot = min(childcols[m], key=itemgetter(1))
-                    if {
-                        a[:-1]
-                        for row in matrix[:-1]
-                        for a in row[:pivot]
-                        if isinstance(a, tuple)
-                    } & {
-                        a[:-1]
-                        for row in matrix[:-1]
-                        for a in row[pivot:]
-                        if isinstance(a, tuple)
-                    }:
-                        crossed.add(m)
-
-                rowidx, i = findcell(m, matrix, startoflevel, childcols)
-                positions.remove(m)
-
-                # block positions where children of this node branch out
-                for _, x in childcols[m]:
-                    matrix[rowidx][x] = corner
-                # assert m == () or matrix[rowidx][i] in (None, corner), (
-                #         matrix[rowidx][i], m, str(tree), ' '.join(sentence))
-                # node itself
-                matrix[rowidx][i] = ids[m]
-                nodes[ids[m]] = tree[m]
-                # add column to the set of children for its parent
-                if m != ():
-                    childcols[m[:-1]].add((rowidx, i))
-        assert len(positions) == 0
-
-        # remove unused columns, right to left
-        for m in range(scale * len(sentence) - 1, -1, -1):
-            if not any(isinstance(row[m], (Tree, int)) for row in matrix):
-                for row in matrix:
-                    del row[m]
-
-        # remove unused rows, reverse
-        matrix = [
-            row
-            for row in reversed(matrix)
-            if not all(a is None or a == vertline for a in row)
-        ]
-
-        # collect coordinates of nodes
-        coords = {}
-        for n, _ in enumerate(matrix):
-            for m, i in enumerate(matrix[n]):
-                if isinstance(i, int) and i >= 0:
-                    coords[i] = n, m
-
-        # move crossed edges last
-        positions = sorted(
-            (a for level in levels.values() for a in level),
-            key=lambda a: a[:-1] in crossed,
-        )
-
-        # collect edges from node to node
-        edges = OrderedDict()
-        for i in reversed(positions):
-            for j, _ in enumerate(tree[i]):
-                edges[ids[i + (j,)]] = ids[i]
-
-        return nodes, coords, edges, highlighted_nodes
-
-    def text(
-        self,
-        nodedist=1,
-        unicodelines=False,
-        html=False,
-        ansi=False,
-        nodecolor="blue",
-        leafcolor="red",
-        funccolor="green",
-        abbreviate=None,
-        maxwidth=16,
-    ):
-        """
-        :return: ASCII art for a discontinuous tree.
-
-        :param unicodelines: whether to use Unicode line drawing characters
-            instead of plain (7-bit) ASCII.
-        :param html: whether to wrap output in html code (default plain text).
-        :param ansi: whether to produce colors with ANSI escape sequences
-            (only effective when html==False).
-        :param leafcolor, nodecolor: specify colors of leaves and phrasal
-            nodes; effective when either html or ansi is True.
-        :param abbreviate: if True, abbreviate labels longer than 5 characters.
-            If integer, abbreviate labels longer than `abbr` characters.
-        :param maxwidth: maximum number of characters before a label starts to
-            wrap; pass None to disable.
-        """
-        if abbreviate == True:
-            abbreviate = 5
-        if unicodelines:
-            horzline = "\u2500"
-            leftcorner = "\u250c"
-            rightcorner = "\u2510"
-            vertline = " \u2502 "
-            tee = horzline + "\u252C" + horzline
-            bottom = horzline + "\u2534" + horzline
-            cross = horzline + "\u253c" + horzline
-            ellipsis = "\u2026"
-        else:
-            horzline = "_"
-            leftcorner = rightcorner = " "
-            vertline = " | "
-            tee = 3 * horzline
-            cross = bottom = "_|_"
-            ellipsis = "."
-
-        def crosscell(cur, x=vertline):
-            """Overwrite center of this cell with a vertical branch."""
-            splitl = len(cur) - len(cur) // 2 - len(x) // 2 - 1
-            lst = list(cur)
-            lst[splitl : splitl + len(x)] = list(x)
-            return "".join(lst)
-
-        result = []
-        matrix = defaultdict(dict)
-        maxnodewith = defaultdict(lambda: 3)
-        maxnodeheight = defaultdict(lambda: 1)
-        maxcol = 0
-        minchildcol = {}
-        maxchildcol = {}
-        childcols = defaultdict(set)
-        labels = {}
-        wrapre = re.compile(
-            "(.{%d,%d}\\b\\W*|.{%d})" % (maxwidth - 4, maxwidth, maxwidth)
-        )
-        # collect labels and coordinates
-        for a in self.nodes:
-            row, column = self.coords[a]
-            matrix[row][column] = a
-            maxcol = max(maxcol, column)
-            label = (
-                self.nodes[a].label()
-                if isinstance(self.nodes[a], Tree)
-                else self.nodes[a]
-            )
-            if abbreviate and len(label) > abbreviate:
-                label = label[:abbreviate] + ellipsis
-            if maxwidth and len(label) > maxwidth:
-                label = wrapre.sub(r"\1\n", label).strip()
-            label = label.split("\n")
-            maxnodeheight[row] = max(maxnodeheight[row], len(label))
-            maxnodewith[column] = max(maxnodewith[column], max(map(len, label)))
-            labels[a] = label
-            if a not in self.edges:
-                continue  # e.g., root
-            parent = self.edges[a]
-            childcols[parent].add((row, column))
-            minchildcol[parent] = min(minchildcol.get(parent, column), column)
-            maxchildcol[parent] = max(maxchildcol.get(parent, column), column)
-        # bottom up level order traversal
-        for row in sorted(matrix, reverse=True):
-            noderows = [
-                ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
-                for _ in range(maxnodeheight[row])
-            ]
-            branchrow = ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
-            for col in matrix[row]:
-                n = matrix[row][col]
-                node = self.nodes[n]
-                text = labels[n]
-                if isinstance(node, Tree):
-                    # draw horizontal branch towards children for this node
-                    if n in minchildcol and minchildcol[n] < maxchildcol[n]:
-                        i, j = minchildcol[n], maxchildcol[n]
-                        a, b = (maxnodewith[i] + 1) // 2 - 1, maxnodewith[j] // 2
-                        branchrow[i] = ((" " * a) + leftcorner).ljust(
-                            maxnodewith[i], horzline
-                        )
-                        branchrow[j] = (rightcorner + (" " * b)).rjust(
-                            maxnodewith[j], horzline
-                        )
-                        for i in range(minchildcol[n] + 1, maxchildcol[n]):
-                            if i == col and any(a == i for _, a in childcols[n]):
-                                line = cross
-                            elif i == col:
-                                line = bottom
-                            elif any(a == i for _, a in childcols[n]):
-                                line = tee
-                            else:
-                                line = horzline
-                            branchrow[i] = line.center(maxnodewith[i], horzline)
-                    else:  # if n and n in minchildcol:
-                        branchrow[col] = crosscell(branchrow[col])
-                text = [a.center(maxnodewith[col]) for a in text]
-                color = nodecolor if isinstance(node, Tree) else leafcolor
-                if isinstance(node, Tree) and node.label().startswith("-"):
-                    color = funccolor
-                if html:
-                    text = [escape(a, quote=False) for a in text]
-                    if n in self.highlight:
-                        text = [f"<font color={color}>{a}</font>" for a in text]
-                elif ansi and n in self.highlight:
-                    text = ["\x1b[%d;1m%s\x1b[0m" % (ANSICOLOR[color], a) for a in text]
-                for x in range(maxnodeheight[row]):
-                    # draw vertical lines in partially filled multiline node
-                    # labels, but only if it's not a frontier node.
-                    noderows[x][col] = (
-                        text[x]
-                        if x < len(text)
-                        else (vertline if childcols[n] else " ").center(
-                            maxnodewith[col], " "
-                        )
-                    )
-            # for each column, if there is a node below us which has a parent
-            # above us, draw a vertical branch in that column.
-            if row != max(matrix):
-                for n, (childrow, col) in self.coords.items():
-                    if n > 0 and self.coords[self.edges[n]][0] < row < childrow:
-                        branchrow[col] = crosscell(branchrow[col])
-                        if col not in matrix[row]:
-                            for noderow in noderows:
-                                noderow[col] = crosscell(noderow[col])
-                branchrow = [
-                    a + ((a[-1] if a[-1] != " " else b[0]) * nodedist)
-                    for a, b in zip(branchrow, branchrow[1:] + [" "])
-                ]
-                result.append("".join(branchrow))
-            result.extend(
-                (" " * nodedist).join(noderow) for noderow in reversed(noderows)
-            )
-        return "\n".join(reversed(result)) + "\n"
-
-    def svg(self, nodecolor="blue", leafcolor="red", funccolor="green"):
-        """
-        :return: SVG representation of a tree.
-        """
-        fontsize = 12
-        hscale = 40
-        vscale = 25
-        hstart = vstart = 20
-        width = max(col for _, col in self.coords.values())
-        height = max(row for row, _ in self.coords.values())
-        result = [
-            '<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
-            'width="%dem" height="%dem" viewBox="%d %d %d %d">'
-            % (
-                width * 3,
-                height * 2.5,
-                -hstart,
-                -vstart,
-                width * hscale + 3 * hstart,
-                height * vscale + 3 * vstart,
-            )
-        ]
-
-        children = defaultdict(set)
-        for n in self.nodes:
-            if n:
-                children[self.edges[n]].add(n)
-
-        # horizontal branches from nodes to children
-        for node in self.nodes:
-            if not children[node]:
-                continue
-            y, x = self.coords[node]
-            x *= hscale
-            y *= vscale
-            x += hstart
-            y += vstart + fontsize // 2
-            childx = [self.coords[c][1] for c in children[node]]
-            xmin = hstart + hscale * min(childx)
-            xmax = hstart + hscale * max(childx)
-            result.append(
-                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
-                'points="%g,%g %g,%g" />' % (xmin, y, xmax, y)
-            )
-            result.append(
-                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
-                'points="%g,%g %g,%g" />' % (x, y, x, y - fontsize // 3)
-            )
-
-        # vertical branches from children to parents
-        for child, parent in self.edges.items():
-            y, _ = self.coords[parent]
-            y *= vscale
-            y += vstart + fontsize // 2
-            childy, childx = self.coords[child]
-            childx *= hscale
-            childy *= vscale
-            childx += hstart
-            childy += vstart - fontsize
-            result += [
-                '\t<polyline style="stroke:white; stroke-width:10; fill:none;"'
-                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y + 5),
-                '\t<polyline style="stroke:black; stroke-width:1; fill:none;"'
-                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y),
-            ]
-
-        # write nodes with coordinates
-        for n, (row, column) in self.coords.items():
-            node = self.nodes[n]
-            x = column * hscale + hstart
-            y = row * vscale + vstart
-            if n in self.highlight:
-                color = nodecolor if isinstance(node, Tree) else leafcolor
-                if isinstance(node, Tree) and node.label().startswith("-"):
-                    color = funccolor
-            else:
-                color = "black"
-            result += [
-                '\t<text style="text-anchor: middle; fill: %s; '
-                'font-size: %dpx;" x="%g" y="%g">%s</text>'
-                % (
-                    color,
-                    fontsize,
-                    x,
-                    y,
-                    escape(
-                        node.label() if isinstance(node, Tree) else node, quote=False
-                    ),
-                )
-            ]
-
-        result += ["</svg>"]
-        return "\n".join(result)
-
-
-def test():
-    """Do some tree drawing tests."""
-
-    def print_tree(n, tree, sentence=None, ansi=True, **xargs):
-        print()
-        print('{}: "{}"'.format(n, " ".join(sentence or tree.leaves())))
-        print(tree)
-        print()
-        drawtree = TreePrettyPrinter(tree, sentence)
-        try:
-            print(drawtree.text(unicodelines=ansi, ansi=ansi, **xargs))
-        except (UnicodeDecodeError, UnicodeEncodeError):
-            print(drawtree.text(unicodelines=False, ansi=False, **xargs))
-
-    from nltk.corpus import treebank
-
-    for n in [0, 1440, 1591, 2771, 2170]:
-        tree = treebank.parsed_sents()[n]
-        print_tree(n, tree, nodedist=2, maxwidth=8)
-    print()
-    print("ASCII version:")
-    print(TreePrettyPrinter(tree).text(nodedist=2))
-
-    tree = Tree.fromstring(
-        "(top (punct 8) (smain (noun 0) (verb 1) (inf (verb 5) (inf (verb 6) "
-        "(conj (inf (pp (prep 2) (np (det 3) (noun 4))) (verb 7)) (inf (verb 9)) "
-        "(vg 10) (inf (verb 11)))))) (punct 12))",
-        read_leaf=int,
-    )
-    sentence = (
-        "Ze had met haar moeder kunnen gaan winkelen ,"
-        " zwemmen of terrassen .".split()
-    )
-    print_tree("Discontinuous tree", tree, sentence, nodedist=2)
+class TreePrettyPrinter(Deprecated, TPP):
+    """Import `TreePrettyPrinter` using `from nltk.tree import TreePrettyPrinter` instead."""
 
 
 __all__ = ["TreePrettyPrinter"]
-
-if __name__ == "__main__":
-    test()
diff --git a/nltk/treetransforms.py b/nltk/treetransforms.py
index 5b89d1a01..d1360618d 100644
--- a/nltk/treetransforms.py
+++ b/nltk/treetransforms.py
@@ -107,232 +107,20 @@ The following is a short tutorial on the available transformations.
 
 """
 
-from nltk.tree import Tree
+from nltk.internals import deprecated
+from nltk.tree.transforms import chomsky_normal_form as cnf
+from nltk.tree.transforms import collapse_unary as cu
+from nltk.tree.transforms import un_chomsky_normal_form as ucnf
+
+chomsky_normal_form = deprecated(
+    "Import using `from nltk.tree import chomsky_normal_form` instead."
+)(cnf)
+un_chomsky_normal_form = deprecated(
+    "Import using `from nltk.tree import un_chomsky_normal_form` instead."
+)(ucnf)
+collapse_unary = deprecated(
+    "Import using `from nltk.tree import collapse_unary` instead."
+)(cu)
 
 
-def chomsky_normal_form(
-    tree, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^"
-):
-    # assume all subtrees have homogeneous children
-    # assume all terminals have no siblings
-
-    # A semi-hack to have elegant looking code below.  As a result,
-    # any subtree with a branching factor greater than 999 will be incorrectly truncated.
-    if horzMarkov is None:
-        horzMarkov = 999
-
-    # Traverse the tree depth-first keeping a list of ancestor nodes to the root.
-    # I chose not to use the tree.treepositions() method since it requires
-    # two traversals of the tree (one to get the positions, one to iterate
-    # over them) and node access time is proportional to the height of the node.
-    # This method is 7x faster which helps when parsing 40,000 sentences.
-
-    nodeList = [(tree, [tree.label()])]
-    while nodeList != []:
-        node, parent = nodeList.pop()
-        if isinstance(node, Tree):
-
-            # parent annotation
-            parentString = ""
-            originalNode = node.label()
-            if vertMarkov != 0 and node != tree and isinstance(node[0], Tree):
-                parentString = "{}<{}>".format(parentChar, "-".join(parent))
-                node.set_label(node.label() + parentString)
-                parent = [originalNode] + parent[: vertMarkov - 1]
-
-            # add children to the agenda before we mess with them
-            for child in node:
-                nodeList.append((child, parent))
-
-            # chomsky normal form factorization
-            if len(node) > 2:
-                childNodes = [child.label() for child in node]
-                nodeCopy = node.copy()
-                node[0:] = []  # delete the children
-
-                curNode = node
-                numChildren = len(nodeCopy)
-                for i in range(1, numChildren - 1):
-                    if factor == "right":
-                        newHead = "{}{}<{}>{}".format(
-                            originalNode,
-                            childChar,
-                            "-".join(
-                                childNodes[i : min([i + horzMarkov, numChildren])]
-                            ),
-                            parentString,
-                        )  # create new head
-                        newNode = Tree(newHead, [])
-                        curNode[0:] = [nodeCopy.pop(0), newNode]
-                    else:
-                        newHead = "{}{}<{}>{}".format(
-                            originalNode,
-                            childChar,
-                            "-".join(
-                                childNodes[max([numChildren - i - horzMarkov, 0]) : -i]
-                            ),
-                            parentString,
-                        )
-                        newNode = Tree(newHead, [])
-                        curNode[0:] = [newNode, nodeCopy.pop()]
-
-                    curNode = newNode
-
-                curNode[0:] = [child for child in nodeCopy]
-
-
-def un_chomsky_normal_form(
-    tree, expandUnary=True, childChar="|", parentChar="^", unaryChar="+"
-):
-    # Traverse the tree-depth first keeping a pointer to the parent for modification purposes.
-    nodeList = [(tree, [])]
-    while nodeList != []:
-        node, parent = nodeList.pop()
-        if isinstance(node, Tree):
-            # if the node contains the 'childChar' character it means that
-            # it is an artificial node and can be removed, although we still need
-            # to move its children to its parent
-            childIndex = node.label().find(childChar)
-            if childIndex != -1:
-                nodeIndex = parent.index(node)
-                parent.remove(parent[nodeIndex])
-                # Generated node was on the left if the nodeIndex is 0 which
-                # means the grammar was left factored.  We must insert the children
-                # at the beginning of the parent's children
-                if nodeIndex == 0:
-                    parent.insert(0, node[0])
-                    parent.insert(1, node[1])
-                else:
-                    parent.extend([node[0], node[1]])
-
-                # parent is now the current node so the children of parent will be added to the agenda
-                node = parent
-            else:
-                parentIndex = node.label().find(parentChar)
-                if parentIndex != -1:
-                    # strip the node name of the parent annotation
-                    node.set_label(node.label()[:parentIndex])
-
-                # expand collapsed unary productions
-                if expandUnary == True:
-                    unaryIndex = node.label().find(unaryChar)
-                    if unaryIndex != -1:
-                        newNode = Tree(
-                            node.label()[unaryIndex + 1 :], [i for i in node]
-                        )
-                        node.set_label(node.label()[:unaryIndex])
-                        node[0:] = [newNode]
-
-            for child in node:
-                nodeList.append((child, node))
-
-
-def collapse_unary(tree, collapsePOS=False, collapseRoot=False, joinChar="+"):
-    """
-    Collapse subtrees with a single child (ie. unary productions)
-    into a new non-terminal (Tree node) joined by 'joinChar'.
-    This is useful when working with algorithms that do not allow
-    unary productions, and completely removing the unary productions
-    would require loss of useful information.  The Tree is modified
-    directly (since it is passed by reference) and no value is returned.
-
-    :param tree: The Tree to be collapsed
-    :type  tree: Tree
-    :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie.
-                        Part-of-Speech tags) since they are always unary productions
-    :type  collapsePOS: bool
-    :param collapseRoot: 'False' (default) will not modify the root production
-                         if it is unary.  For the Penn WSJ treebank corpus, this corresponds
-                         to the TOP -> productions.
-    :type collapseRoot: bool
-    :param joinChar: A string used to connect collapsed node values (default = "+")
-    :type  joinChar: str
-    """
-
-    if collapseRoot == False and isinstance(tree, Tree) and len(tree) == 1:
-        nodeList = [tree[0]]
-    else:
-        nodeList = [tree]
-
-    # depth-first traversal of tree
-    while nodeList != []:
-        node = nodeList.pop()
-        if isinstance(node, Tree):
-            if (
-                len(node) == 1
-                and isinstance(node[0], Tree)
-                and (collapsePOS == True or isinstance(node[0, 0], Tree))
-            ):
-                node.set_label(node.label() + joinChar + node[0].label())
-                node[0:] = [child for child in node[0]]
-                # since we assigned the child's children to the current node,
-                # evaluate the current node again
-                nodeList.append(node)
-            else:
-                for child in node:
-                    nodeList.append(child)
-
-
-#################################################################
-# Demonstration
-#################################################################
-
-
-def demo():
-    """
-    A demonstration showing how each tree transform can be used.
-    """
-
-    from copy import deepcopy
-
-    from nltk import tree, treetransforms
-    from nltk.draw.tree import draw_trees
-
-    # original tree from WSJ bracketed text
-    sentence = """(TOP
-  (S
-    (S
-      (VP
-        (VBN Turned)
-        (ADVP (RB loose))
-        (PP
-          (IN in)
-          (NP
-            (NP (NNP Shane) (NNP Longman) (POS 's))
-            (NN trading)
-            (NN room)))))
-    (, ,)
-    (NP (DT the) (NN yuppie) (NNS dealers))
-    (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
-    (. .)))"""
-    t = tree.Tree.fromstring(sentence, remove_empty_top_bracketing=True)
-
-    # collapse subtrees with only one child
-    collapsedTree = deepcopy(t)
-    treetransforms.collapse_unary(collapsedTree)
-
-    # convert the tree to CNF
-    cnfTree = deepcopy(collapsedTree)
-    treetransforms.chomsky_normal_form(cnfTree)
-
-    # convert the tree to CNF with parent annotation (one level) and horizontal smoothing of order two
-    parentTree = deepcopy(collapsedTree)
-    treetransforms.chomsky_normal_form(parentTree, horzMarkov=2, vertMarkov=1)
-
-    # convert the tree back to its original form (used to make CYK results comparable)
-    original = deepcopy(parentTree)
-    treetransforms.un_chomsky_normal_form(original)
-
-    # convert tree back to bracketed text
-    sentence2 = original.pprint()
-    print(sentence)
-    print(sentence2)
-    print("Sentences the same? ", sentence == sentence2)
-
-    draw_trees(t, collapsedTree, cnfTree, parentTree, original)
-
-
-if __name__ == "__main__":
-    demo()
-
 __all__ = ["chomsky_normal_form", "un_chomsky_normal_form", "collapse_unary"]


Test patch:
diff --git a/nltk/test/corpus.doctest b/nltk/test/corpus.doctest
index 47a6fea2f..9f283d2c0 100644
--- a/nltk/test/corpus.doctest
+++ b/nltk/test/corpus.doctest
@@ -1439,12 +1439,12 @@ constructor will first call its base class's constructor, and then
 store the customizable parameters.  For example, the
 `ConllChunkCorpusReader`\ 's constructor is defined as follows:
 
-    def __init__(self, root, fileids, chunk_types, encoding='utf8',
-                 tagset=None, separator=None):
-        ConllCorpusReader.__init__(
-                self, root, fileids, ('words', 'pos', 'chunk'),
-                chunk_types=chunk_types, encoding=encoding,
-                tagset=tagset, separator=separator)
+    >>> def __init__(self, root, fileids, chunk_types, encoding='utf8',
+    ...              tagset=None, separator=None):
+    ...     ConllCorpusReader.__init__(
+    ...             self, root, fileids, ('words', 'pos', 'chunk'),
+    ...             chunk_types=chunk_types, encoding=encoding,
+    ...             tagset=tagset, separator=separator)
 
 If your corpus reader does not implement any customization parameters,
 then you can often just inherit the base class's constructor.
diff --git a/nltk/test/featstruct.doctest b/nltk/test/featstruct.doctest
index 78b23c5ee..9d1c7c794 100644
--- a/nltk/test/featstruct.doctest
+++ b/nltk/test/featstruct.doctest
@@ -679,6 +679,7 @@ Special features:
     [ tense   = 'past'            ]
 
 Here the slash feature gets coerced:
+
     >>> FeatStruct('[*slash*=a, x=b, *type*="NP"]')
     NP[x='b']/a[]
 
diff --git a/nltk/test/gensim.doctest b/nltk/test/gensim.doctest
index f9215b0bb..3a418c862 100644
--- a/nltk/test/gensim.doctest
+++ b/nltk/test/gensim.doctest
@@ -31,6 +31,7 @@ It might take some time to train the model. So, after it is trained, it can be s
     >>> new_model = gensim.models.Word2Vec.load('brown.embedding')
 
 The model will be the list of words with their embedding. We can easily get the vector representation of a word.
+
     >>> len(new_model['university'])
     100
 
diff --git a/nltk/test/internals.doctest b/nltk/test/internals.doctest
index 38688c327..491bf1576 100644
--- a/nltk/test/internals.doctest
+++ b/nltk/test/internals.doctest
@@ -150,6 +150,7 @@ Test valid scenarios
     ('valid string', 14)
 
 Now test invalid scenarios
+
     >>> read_str("should error", 0)
     Traceback (most recent call last):
     ...
diff --git a/nltk/test/lm.doctest b/nltk/test/lm.doctest
index b76cfb4a7..a77954add 100644
--- a/nltk/test/lm.doctest
+++ b/nltk/test/lm.doctest
@@ -121,6 +121,7 @@ Reproducing setup akin to this comment:
 https://github.com/nltk/nltk/issues/380#issue-12879030
 
 For speed take only the first 100 sentences of reuters. Shouldn't affect the test.
+
     >>> from nltk.corpus import reuters
     >>> sents = reuters.sents()[:100]
     >>> ngram_order = 3
diff --git a/nltk/test/nonmonotonic.doctest b/nltk/test/nonmonotonic.doctest
index 1929878ea..62980ea5f 100644
--- a/nltk/test/nonmonotonic.doctest
+++ b/nltk/test/nonmonotonic.doctest
@@ -228,6 +228,7 @@ Default Reasoning
     >>> premises = []
 
 define the taxonomy
+
     >>> premises.append(read_expr(r'all x.(elephant(x)        -> animal(x))'))
     >>> premises.append(read_expr(r'all x.(bird(x)            -> animal(x))'))
     >>> premises.append(read_expr(r'all x.(dove(x)            -> bird(x))'))
@@ -235,21 +236,25 @@ define the taxonomy
     >>> premises.append(read_expr(r'all x.(flying_ostrich(x)  -> ostrich(x))'))
 
 default the properties using abnormalities
+
     >>> premises.append(read_expr(r'all x.((animal(x)  & -Ab1(x)) -> -fly(x))')) #normal animals don't fly
     >>> premises.append(read_expr(r'all x.((bird(x)    & -Ab2(x)) -> fly(x))'))  #normal birds fly
     >>> premises.append(read_expr(r'all x.((ostrich(x) & -Ab3(x)) -> -fly(x))')) #normal ostriches don't fly
 
 specify abnormal entities
+
     >>> premises.append(read_expr(r'all x.(bird(x)           -> Ab1(x))')) #flight
     >>> premises.append(read_expr(r'all x.(ostrich(x)        -> Ab2(x))')) #non-flying bird
     >>> premises.append(read_expr(r'all x.(flying_ostrich(x) -> Ab3(x))')) #flying ostrich
 
 define entities
+
     >>> premises.append(read_expr(r'elephant(el)'))
     >>> premises.append(read_expr(r'dove(do)'))
     >>> premises.append(read_expr(r'ostrich(os)'))
 
 print the augmented assumptions list
+
     >>> prover = Prover9Command(None, premises)
     >>> command = UniqueNamesProver(ClosedWorldProver(prover))
     >>> for a in command.assumptions(): print(a) # doctest: +SKIP
diff --git a/nltk/test/portuguese_en.doctest b/nltk/test/portuguese_en.doctest
index ee1641e64..e490d4cee 100644
--- a/nltk/test/portuguese_en.doctest
+++ b/nltk/test/portuguese_en.doctest
@@ -240,6 +240,7 @@ journalistic texts extracted from ten sections of
 the daily newspaper *Folha de Sao Paulo*, 1994.
 
 We can access this corpus as a sequence of words or tagged words as follows:
+
     >>> import nltk.corpus
     >>> nltk.corpus.mac_morpho.words()
     ['Jersei', 'atinge', 'm\xe9dia', 'de', 'Cr$', '1,4', ...]
diff --git a/nltk/test/probability.doctest b/nltk/test/probability.doctest
index a955a5f87..952c03c02 100644
--- a/nltk/test/probability.doctest
+++ b/nltk/test/probability.doctest
@@ -194,6 +194,7 @@ our testing accordingly.
     ...         for sent in corpus[:100]]
 
 We will then need to redefine the rest of the training/testing variables
+
     >>> tag_set = unique_list(tag for sent in corpus for (word,tag) in sent)
     >>> len(tag_set)
     906
diff --git a/nltk/test/resolution.doctest b/nltk/test/resolution.doctest
index 1177309bc..bea380f5c 100644
--- a/nltk/test/resolution.doctest
+++ b/nltk/test/resolution.doctest
@@ -173,6 +173,7 @@ Test proof()
 Question Answering
 ------------------
 One answer
+
     >>> p1 = read_expr('father_of(art,john)')
     >>> p2 = read_expr('father_of(bob,kim)')
     >>> p3 = read_expr('all x.all y.(father_of(x,y) -> parent_of(x,y))')
@@ -194,6 +195,7 @@ One answer
     <BLANKLINE>
 
 Multiple answers
+
     >>> p1 = read_expr('father_of(art,john)')
     >>> p2 = read_expr('mother_of(ann,john)')
     >>> p3 = read_expr('all x.all y.(father_of(x,y) -> parent_of(x,y))')
diff --git a/nltk/test/tokenize.doctest b/nltk/test/tokenize.doctest
index a5e86d44e..0e3a0749e 100644
--- a/nltk/test/tokenize.doctest
+++ b/nltk/test/tokenize.doctest
@@ -3,10 +3,10 @@
 
     >>> from nltk.tokenize import *
 
-Regression Tests: Treebank Tokenizer
+Regression Tests: NLTKWordTokenizer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Some test strings.
+Tokenizing some test strings.
 
     >>> s1 = "On a $50,000 mortgage of 30 years at 8 percent, the monthly payment would be $366.88."
     >>> word_tokenize(s1)
@@ -42,6 +42,35 @@ Some test strings.
     >>> word_tokenize(s11)
     ['It', "'s", 'more', "'n", 'enough', '.']
 
+Gathering the spans of the tokenized strings.
+
+    >>> s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
+    >>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
+    ... (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
+    ... (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
+    ... (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+    >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
+    True
+    >>> expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+    ... 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
+    ... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+    >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
+    True
+
+    >>> s = '''I said, "I'd like to buy some ''good muffins" which cost $3.88\n each in New (York)."'''
+    >>> expected = [(0, 1), (2, 6), (6, 7), (8, 9), (9, 10), (10, 12),
+    ... (13, 17), (18, 20), (21, 24), (25, 29), (30, 32), (32, 36),
+    ... (37, 44), (44, 45), (46, 51), (52, 56), (57, 58), (58, 62),
+    ... (64, 68), (69, 71), (72, 75), (76, 77), (77, 81), (81, 82),
+    ... (82, 83), (83, 84)]
+    >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
+    True
+    >>> expected = ['I', 'said', ',', '"', 'I', "'d", 'like', 'to',
+    ... 'buy', 'some', "''", "good", 'muffins', '"', 'which', 'cost',
+    ... '$', '3.88', 'each', 'in', 'New', '(', 'York', ')', '.', '"']
+    >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
+    True
+
 Testing improvement made to the TreebankWordTokenizer
 
     >>> sx1 = '\xabNow that I can do.\xbb'
diff --git a/nltk/test/tree.doctest b/nltk/test/tree.doctest
index a47927328..feea8201b 100644
--- a/nltk/test/tree.doctest
+++ b/nltk/test/tree.doctest
@@ -113,7 +113,7 @@ type:
     >>> print(tree)
     (VP (V enjoyed) (NP my cookie))
     >>> print(type(tree))
-    <class 'nltk.tree.ImmutableTree'>
+    <class 'nltk.tree.immutable.ImmutableTree'>
     >>> tree[1] = 'x'
     Traceback (most recent call last):
       . . .
@@ -345,7 +345,7 @@ Parented trees can be created from strings using the classmethod
     >>> print(ptree)
     (VP (VERB saw) (NP (DET the) (NOUN dog)))
     >>> print(type(ptree))
-    <class 'nltk.tree.ParentedTree'>
+    <class 'nltk.tree.parented.ParentedTree'>
 
 Parented trees can also be created by using the classmethod
 `ParentedTree.convert` to convert another type of tree to a parented
@@ -356,7 +356,7 @@ tree:
     >>> print(ptree)
     (VP (VERB saw) (NP (DET the) (NOUN dog)))
     >>> print(type(ptree))
-    <class 'nltk.tree.ParentedTree'>
+    <class 'nltk.tree.parented.ParentedTree'>
 
 .. clean-up:
 
@@ -802,13 +802,59 @@ Test that a tree can not be given multiple parents:
 
 [more to be written]
 
+Shallow copying can be tricky for Tree and several of its subclasses.
+For shallow copies of Tree, only the root node is reconstructed, while
+all the children are shared between the two trees. Modify the children
+of one tree - and the shallowly copied tree will also update.
+
+    >>> from nltk.tree import Tree, ParentedTree, MultiParentedTree
+    >>> tree = Tree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> copy_tree = tree.copy(deep=False)
+    >>> tree == copy_tree # Ensure identical labels and nodes
+    True
+    >>> id(copy_tree[0]) == id(tree[0]) # Ensure shallow copy - the children are the same objects in memory
+    True
+
+For ParentedTree objects, this behaviour is not possible. With a shallow
+copy, the children of the root node would be reused for both the original
+and the shallow copy. For this to be possible, some children would need
+to have multiple parents. As this is forbidden for ParentedTree objects,
+attempting to make a shallow copy will cause a warning, and a deep copy
+is made instead.
+
+    >>> ptree = ParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> copy_ptree = ptree.copy(deep=False)
+    >>> copy_ptree == ptree # Ensure identical labels and nodes
+    True
+    >>> id(copy_ptree[0]) != id(ptree[0]) # Shallow copying isn't supported - it defaults to deep copy.
+    True
+
+For MultiParentedTree objects, the issue of only allowing one parent that
+can be seen for ParentedTree objects is no more. Shallow copying a
+MultiParentedTree gives the children of the root node two parents:
+the original and the newly copied root.
+
+    >>> mptree = MultiParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> len(mptree[0].parents())
+    1
+    >>> copy_mptree = mptree.copy(deep=False)
+    >>> copy_mptree == mptree # Ensure identical labels and nodes
+    True
+    >>> len(mptree[0].parents())
+    2
+    >>> len(copy_mptree[0].parents())
+    2
+
+Shallow copying a MultiParentedTree is similar to creating a second root
+which is identically labeled as the root on which the copy method was called.
+
 
 ImmutableParentedTree Regression Tests
 --------------------------------------
 
     >>> iptree = ImmutableParentedTree.convert(ptree)
     >>> type(iptree)
-    <class 'nltk.tree.ImmutableParentedTree'>
+    <class 'nltk.tree.immutable.ImmutableParentedTree'>
     >>> del iptree[0]
     Traceback (most recent call last):
       . . .
@@ -1110,7 +1156,7 @@ ImmutableMultiParentedTree Regression Tests
 
     >>> imptree = ImmutableMultiParentedTree.convert(mptree)
     >>> type(imptree)
-    <class 'nltk.tree.ImmutableMultiParentedTree'>
+    <class 'nltk.tree.immutable.ImmutableMultiParentedTree'>
     >>> del imptree[0]
     Traceback (most recent call last):
       . . .
@@ -1137,7 +1183,7 @@ ProbabilisticTree Regression Tests
 
     >>> imprtree = ImmutableProbabilisticTree.convert(prtree)
     >>> type(imprtree)
-    <class 'nltk.tree.ImmutableProbabilisticTree'>
+    <class 'nltk.tree.immutable.ImmutableProbabilisticTree'>
     >>> del imprtree[0]
     Traceback (most recent call last):
       . . .
@@ -1155,3 +1201,23 @@ This used to discard the ``(B b)`` subtree (fixed in svn 6270):
 
     >>> print(Tree.fromstring('((A a) (B b))'))
     ( (A a) (B b))
+
+Pickling ParentedTree instances didn't work for Python 3.7 onwards (See #2478)
+
+    >>> import pickle
+    >>> tree = ParentedTree.fromstring('(S (NN x) (NP x) (NN x))')
+    >>> print(tree)
+    (S (NN x) (NP x) (NN x))
+
+    >>> pickled = pickle.dumps(tree)
+    >>> tree_loaded = pickle.loads(pickled)
+    >>> print(tree_loaded)
+    (S (NN x) (NP x) (NN x))
+
+ParentedTree used to be impossible to (deep)copy. (See #1324)
+
+    >>> from nltk.tree import ParentedTree
+    >>> import copy
+    >>> tree = ParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> tree == copy.deepcopy(tree) == copy.copy(tree) == tree.copy(deep=True) == tree.copy()
+    True
diff --git a/nltk/test/treeprettyprinter.doctest b/nltk/test/treeprettyprinter.doctest
index cea049c98..9f11af21e 100644
--- a/nltk/test/treeprettyprinter.doctest
+++ b/nltk/test/treeprettyprinter.doctest
@@ -1,12 +1,11 @@
 .. Copyright (C) 2001-2021 NLTK Project
 .. For license information, see LICENSE.TXT
 
-========================================================
- Unit tests for nltk.treeprettyprinter.TreePrettyPrinter
-========================================================
+=========================================================
+ Unit tests for nltk.tree.prettyprinter.TreePrettyPrinter
+=========================================================
 
-    >>> from nltk.tree import Tree
-    >>> from nltk.treeprettyprinter import TreePrettyPrinter
+    >>> from nltk.tree import Tree, TreePrettyPrinter
 
 Tree nr 2170 from nltk.corpus.treebank:
 
@@ -124,3 +123,55 @@ A discontinuous tree:
     noun verb prep det       noun   verb  verb       verb   punct   verb   vg    verb   punct
      │    │    │    │         │      │     │          │       │      │     │      │       │
      Ze  had  met  haar     moeder kunnen gaan     winkelen   ,   zwemmen  of terrassen   .
+
+Importing TreePrettyPrinter
+---------------------------
+
+First of all, a simple tree will be constructed::
+
+  >>> from nltk.tree import Tree
+  >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))')
+
+We'll use this sample tree to show that the method of importing `TreePrettyPrinter` work correctly:
+
+- Recommended::
+
+    >>> from nltk.tree import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+- Alternative but valid options::
+
+    >>> from nltk import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+    >>> from nltk.tree.prettyprinter import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+- Deprecated, do not use::
+
+    >>> from nltk.treeprettyprinter import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+  This method will throw a DeprecationWarning::
+
+    Import `TreePrettyPrinter` using `from nltk.tree import TreePrettyPrinter` instead.
diff --git a/nltk/test/treetransforms.doctest b/nltk/test/treetransforms.doctest
index a3870138d..411499340 100644
--- a/nltk/test/treetransforms.doctest
+++ b/nltk/test/treetransforms.doctest
@@ -6,8 +6,7 @@ Unit tests for the TreeTransformation class
 -------------------------------------------
 
     >>> from copy import deepcopy
-    >>> from nltk.tree import *
-    >>> from nltk.treetransforms import *
+    >>> from nltk.tree import Tree, collapse_unary, chomsky_normal_form, un_chomsky_normal_form
 
     >>> tree_string = "(TOP (S (S (VP (VBN Turned) (ADVP (RB loose)) (PP (IN in) (NP (NP (NNP Shane) (NNP Longman) (POS 's)) (NN trading) (NN room))))) (, ,) (NP (DT the) (NN yuppie) (NNS dealers)) (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right)))) (. .)))"
 
diff --git a/nltk/test/unit/test_distance.py b/nltk/test/unit/test_distance.py
index 23a5b9c4f..96d814d0b 100644
--- a/nltk/test/unit/test_distance.py
+++ b/nltk/test/unit/test_distance.py
@@ -94,6 +94,12 @@ class TestEditDistance:
             # (but cost 5 if substitution_cost=2)
             ("kitten", "sitting", 1, (3, 3)),
             ("kitten", "sitting", 2, (5, 5)),
+            #
+            # duplicated letter
+            # e.g. "duplicated" -D-> "duplicated"
+            ("duplicated", "duuplicated", 1, (1, 1)),
+            ("duplicated", "duuplicated", 2, (1, 1)),
+            ("very duplicated", "very duuplicateed", 2, (2, 2)),
         ],
     )
     def test_with_transpositions(
diff --git a/nltk/test/wordnet.doctest b/nltk/test/wordnet.doctest
index e1ed56657..d885839ab 100644
--- a/nltk/test/wordnet.doctest
+++ b/nltk/test/wordnet.doctest
@@ -6,6 +6,7 @@ WordNet Interface
 =================
 
 WordNet is just another NLTK corpus reader, and can be imported like this:
+
     >>> from nltk.corpus import wordnet
 
 For more compact code, we recommend:
@@ -53,11 +54,10 @@ WordNet, using ISO-639 language codes.
     >>> wn.synsets(b'\xe7\x8a\xac'.decode('utf-8'), lang='jpn')
     [Synset('dog.n.01'), Synset('spy.n.01')]
 
-    wn.synset('spy.n.01').lemma_names('jpn')
-    ['\u3044\u306c', '\u307e\u308f\u3057\u8005', '\u30b9\u30d1\u30a4', '\u56de\u3057\u8005',
-    '\u56de\u8005', '\u5bc6\u5075', '\u5de5\u4f5c\u54e1', '\u5efb\u3057\u8005',
-    '\u5efb\u8005', '\u63a2', '\u63a2\u308a', '\u72ac', '\u79d8\u5bc6\u635c\u67fb\u54e1',
-    '\u8adc\u5831\u54e1', '\u8adc\u8005', '\u9593\u8005', '\u9593\u8adc', '\u96a0\u5bc6']
+    >>> wn.synset('spy.n.01').lemma_names('jpn')
+    ['いぬ', 'スパイ', '回者', '回し者', '密偵', '工作員',
+    '廻者', '廻し者', '探', '探り', '犬', '秘密捜査員',
+    'まわし者', '諜報員', '諜者', '間者', '間諜', '隠密']
 
     >>> wn.synset('dog.n.01').lemma_names('ita')
     ['cane', 'Canis_familiaris']
@@ -68,8 +68,8 @@ WordNet, using ISO-639 language codes.
     [Lemma('dog.n.01.hund'), Lemma('dog.n.01.k\xf8ter'),
     Lemma('dog.n.01.vovhund'), Lemma('dog.n.01.vovse')]
 
-    sorted(wn.synset('dog.n.01').lemmas('por'))
-	[Lemma('dog.n.01.cachorra'), Lemma('dog.n.01.cachorro'), Lemma('dog.n.01.cadela'), Lemma('dog.n.01.c\xe3o')]
+    >>> sorted(wn.synset('dog.n.01').lemmas('por'))
+    [Lemma('dog.n.01.cachorra'), Lemma('dog.n.01.cachorro'), Lemma('dog.n.01.cadela'), Lemma('dog.n.01.c\xe3o')]
 
     >>> dog_lemma = wn.lemma(b'dog.n.01.c\xc3\xa3o'.decode('utf-8'), lang='por')
     >>> dog_lemma
@@ -450,6 +450,7 @@ as a base form for a POS for which that word is not defined:
     >>> wn.morphy('book', wn.NOUN)
     'book'
     >>> wn.morphy('book', wn.ADJ)
+    >>>
 
 Bug 160: wup_similarity breaks when the two synsets have no common hypernym
 
@@ -458,8 +459,9 @@ Bug 160: wup_similarity breaks when the two synsets have no common hypernym
     >>> t.wup_similarity(m)
     0.631...
 
-    Issue #2278: wup_similarity not commutative when comparing a noun and a verb.
-    Patch #2650 resolved this error. As a result, the output of the following use of wup_similarity no longer returns None.
+Issue #2278: wup_similarity not commutative when comparing a noun and a verb.
+Patch #2650 resolved this error. As a result, the output of the following use of wup_similarity no longer returns None.
+
     >>> t = wn.synsets('titan')[1]
     >>> s = wn.synsets('say', wn.VERB)[0]
     >>> t.wup_similarity(s)
@@ -598,6 +600,7 @@ Bug https://github.com/nltk/nltk/issues/1641: Non-English lemmas containing capi
     [Lemma('united_kingdom.n.01.Londres'), Lemma('london.n.01.Londres'), Lemma('london.n.02.Londres')]
 
 Patch-1 https://github.com/nltk/nltk/pull/2065  Adding 3 functions (relations) to WordNet class
+
     >>> wn.synsets("computer_science")[0].in_topic_domains()[2]
     Synset('access_time.n.01')
     >>> wn.synsets("France")[0].in_region_domains()[18]
@@ -758,6 +761,27 @@ classified.a.02):
       [Synset('restricted.a.01'), [Synset('classified.a.02')]]]]
 
 
+----------------------------------------------------------------
+Loading alternative Wordnet versions
+----------------------------------------------------------------
+
+    >>> print("Wordnet {}".format(wn.get_version()))
+    Wordnet 3.0
+
+    >>> from nltk.corpus import wordnet31 as wn31
+    >>> print("Wordnet {}".format(wn31.get_version()))
+    Wordnet 3.1
+
+    >>> print(wn.synset('restrain.v.01').hyponyms())
+    [Synset('confine.v.03'), Synset('control.v.02'), Synset('hold.v.36'), Synset('inhibit.v.04')]
+
+    >>> print(wn31.synset('restrain.v.01').hyponyms())
+    [Synset('enchain.v.01'), Synset('fetter.v.01'), Synset('ground.v.02'), Synset('impound.v.02'), Synset('pen_up.v.01'), Synset('pinion.v.01'), Synset('pound.v.06'), Synset('tie_down.v.01')]
+
+    >>> print(wn31.synset('restrain.v.04').hyponyms())
+    [Synset('baffle.v.03'), Synset('confine.v.02'), Synset('control.v.02'), Synset('hold.v.36'), Synset('rule.v.07'), Synset('swallow.v.06'), Synset('wink.v.04')]
+
+
 -------------
 Teardown test
 -------------