Repository: nltk/nltk

Issue:
Add `CITATION.cff` file to repository 
Github has a built-in support for citation. Can you please consider adding  CITATTION.cff file to your repository so we can easy cite your work?


Info:
https://citation-file-format.github.io/
https://twitter.com/natfriedman/status/1420122675813441540?lang=en

Discussion:
iliakur: After skimming, an idea for software title: **Natural Language ToolKit (NLTK)**. This is commonly how acronyms are introduced in prose.
tomaarsen: That sounds like a good middle-ground.

Patch:
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 4af8d4d98..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-sudo: false
-
-language: python
-
-cache:
-  apt: true
-  directories:
-  - $HOME/.cache/pip
-  - $HOME/.ccache
-  - $HOME/.pip-cache
-  - $HOME/third
-  - $HOME/nltk_data
-
-dist: xenial
-
-addons:
-  apt:
-    packages:
-      - expect-dev # for unbuffer, see https://github.com/travis-ci/travis-ci/issues/7967
-
-matrix:
-  include:
-    - python: '3.6'
-      env: TOXENV="py-travis"
-
-    - python: '3.7'
-      env: TOXENV="py-travis"
-
-    - python: '3.8'
-      env: TOXENV="py-travis"
-
-    - python: '3.9'
-      env: TOXENV="py-travis"
-
-    - python: '3.9' # Only runs checks on 3rd party for latest Python distribution
-      env: TOXENV="py-travis-third-party"
-
-before_install:
-  - source tools/travis/pre-install.sh # Checks Java and Python versions.
-  - chmod +x tools/travis/coverage-pylint.sh
-  - chmod +x tools/travis/third-party.sh
-
-install:
-  - source tools/travis/install.sh
-  #- source tools/travis/third-party.sh
-  - pip install tox
-
-script:
-  - unbuffer tox -e $TOXENV tox.ini -vv
-
-after_success:
-  - tools/travis/travis-ci-commenter.sh
diff --git a/AUTHORS.md b/AUTHORS.md
index 5b7546d43..04592e11c 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -285,6 +285,7 @@
 - Saibo Geng <https://github.com/Saibo-creator>
 - Ahmet Yildirim <https://github.com/RnDevelover>
 - Yuta Nakamura <https://github.com/yutanakamura-tky>
+- Adam Hawley <https://github.com/adamjhawley>
 
 ## Others whose work we've taken and included in NLTK, but who didn't directly contribute it:
 
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 000000000..e8e96f762
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,43 @@
+cff-version: 1.2.0
+title: >-
+  Natural Language ToolKit (NLTK)
+message: >-
+  Please cite this software using the metadata from
+  'preferred-citation'.
+type: software
+authors:
+  - name: "NLTK Team"
+    email: "nltk.team@gmail.com"
+repository-code: "https://github.com/nltk/nltk"
+url: "https://www.nltk.org"
+license: Apache-2.0
+keywords:
+  - "NLP"
+  - "CL"
+  - "natural language processing"
+  - "computational linguistics"
+  - "parsing"
+  - "tagging"
+  - "tokenizing"
+  - "syntax"
+  - "linguistics"
+  - "language"
+  - "natural language"
+  - "text analytics"
+preferred-citation:
+  title: >-
+    Natural Language Processing with Python: Analyzing
+    Text with the Natural Language Toolkit
+  type: book
+  authors:
+    - given-names: Steven
+      family-names: Bird
+      orcid: https://orcid.org/0000-0003-3782-7733
+    - given-names: Ewan
+      family-names: Klein
+      orcid: https://orcid.org/0000-0002-0520-8447
+    - given-names: Edward
+      family-names: Loper
+  year: 2009
+  publisher:
+    name: "O'Reilly Media, Inc."
diff --git a/INSTALL.txt b/INSTALL.txt
deleted file mode 100644
index cba3d199a..000000000
--- a/INSTALL.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-To install NLTK, run setup.py from an administrator account, e.g.:
-
-    sudo python setup.py install
-
-For full installation instructions, please see https://www.nltk.org/install.html
diff --git a/NOTICE.txt b/NOTICE.txt
deleted file mode 100644
index 51eb38d61..000000000
--- a/NOTICE.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Natural Language Toolkit (NLTK)
-Copyright (C) 2001-2021 NLTK Project
diff --git a/jenkins-job-config.xml b/jenkins-job-config.xml
deleted file mode 100644
index 603d719dd..000000000
--- a/jenkins-job-config.xml
+++ /dev/null
@@ -1,341 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<matrix-project>
-  <actions/>
-  <description></description>
-  <keepDependencies>false</keepDependencies>
-  <properties>
-    <com.coravy.hudson.plugins.github.GithubProjectProperty plugin="github@1.4">
-      <projectUrl>https://github.com/nltk/nltk/</projectUrl>
-    </com.coravy.hudson.plugins.github.GithubProjectProperty>
-  </properties>
-  <scm class="hudson.plugins.git.GitSCM" plugin="git@1.1.26">
-    <configVersion>2</configVersion>
-    <userRemoteConfigs>
-      <hudson.plugins.git.UserRemoteConfig>
-        <name></name>
-        <refspec></refspec>
-        <url>git://github.com/nltk/nltk.git</url>
-      </hudson.plugins.git.UserRemoteConfig>
-    </userRemoteConfigs>
-    <branches>
-      <hudson.plugins.git.BranchSpec>
-        <name>origin/develop</name>
-      </hudson.plugins.git.BranchSpec>
-    </branches>
-    <disableSubmodules>false</disableSubmodules>
-    <recursiveSubmodules>false</recursiveSubmodules>
-    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
-    <authorOrCommitter>false</authorOrCommitter>
-    <clean>false</clean>
-    <wipeOutWorkspace>false</wipeOutWorkspace>
-    <pruneBranches>false</pruneBranches>
-    <remotePoll>false</remotePoll>
-    <ignoreNotifyCommit>false</ignoreNotifyCommit>
-    <useShallowClone>false</useShallowClone>
-    <buildChooser class="hudson.plugins.git.util.DefaultBuildChooser"/>
-    <gitTool>Default</gitTool>
-    <submoduleCfg class="list"/>
-    <relativeTargetDir></relativeTargetDir>
-    <reference></reference>
-    <excludedRegions></excludedRegions>
-    <excludedUsers></excludedUsers>
-    <gitConfigName></gitConfigName>
-    <gitConfigEmail></gitConfigEmail>
-    <skipTag>false</skipTag>
-    <includedRegions></includedRegions>
-    <scmName></scmName>
-  </scm>
-  <canRoam>true</canRoam>
-  <disabled>false</disabled>
-  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
-  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
-  <triggers class="vector">
-    <com.cloudbees.jenkins.GitHubPushTrigger plugin="github@1.4">
-      <spec></spec>
-    </com.cloudbees.jenkins.GitHubPushTrigger>
-    <hudson.triggers.SCMTrigger>
-      <spec>H/5 * * * *</spec>
-      <ignorePostCommitHooks>false</ignorePostCommitHooks>
-    </hudson.triggers.SCMTrigger>
-  </triggers>
-  <concurrentBuild>false</concurrentBuild>
-  <axes>
-    <jenkins.plugins.shiningpanda.matrix.ToxAxis plugin="shiningpanda@0.16">
-      <name>TOXENV</name>
-      <values>
-        <string>py32-jenkins</string>
-      </values>
-    </jenkins.plugins.shiningpanda.matrix.ToxAxis>
-  </axes>
-  <builders>
-    <jenkins.plugins.shiningpanda.builders.ToxBuilder plugin="shiningpanda@0.16">
-      <toxIni>tox.ini</toxIni>
-      <recreate>false</recreate>
-    </jenkins.plugins.shiningpanda.builders.ToxBuilder>
-  </builders>
-  <publishers>
-    <hudson.plugins.cobertura.CoberturaPublisher plugin="cobertura@1.7.1">
-      <coberturaReportFile>**/coverage.xml</coberturaReportFile>
-      <onlyStable>false</onlyStable>
-      <failUnhealthy>false</failUnhealthy>
-      <failUnstable>false</failUnstable>
-      <autoUpdateHealth>false</autoUpdateHealth>
-      <autoUpdateStability>false</autoUpdateStability>
-      <healthyTarget>
-        <targets class="enum-map" enum-type="hudson.plugins.cobertura.targets.CoverageMetric">
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>CONDITIONAL</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>7000000</int>
-          </entry>
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>LINE</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>8000000</int>
-          </entry>
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>METHOD</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>8000000</int>
-          </entry>
-        </targets>
-      </healthyTarget>
-      <unhealthyTarget>
-        <targets class="enum-map" enum-type="hudson.plugins.cobertura.targets.CoverageMetric">
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>CONDITIONAL</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>0</int>
-          </entry>
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>LINE</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>0</int>
-          </entry>
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>METHOD</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>0</int>
-          </entry>
-        </targets>
-      </unhealthyTarget>
-      <failingTarget>
-        <targets class="enum-map" enum-type="hudson.plugins.cobertura.targets.CoverageMetric">
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>CONDITIONAL</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>0</int>
-          </entry>
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>LINE</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>0</int>
-          </entry>
-          <entry>
-            <hudson.plugins.cobertura.targets.CoverageMetric>METHOD</hudson.plugins.cobertura.targets.CoverageMetric>
-            <int>0</int>
-          </entry>
-        </targets>
-      </failingTarget>
-      <sourceEncoding>ASCII</sourceEncoding>
-    </hudson.plugins.cobertura.CoberturaPublisher>
-    <hudson.tasks.junit.JUnitResultArchiver>
-      <testResults>**/coverage_scrubbed.xml</testResults>
-      <keepLongStdio>false</keepLongStdio>
-      <testDataPublishers/>
-    </hudson.tasks.junit.JUnitResultArchiver>
-    <hudson.plugins.violations.ViolationsPublisher plugin="violations@0.7.11">
-      <config>
-        <suppressions class="tree-set">
-          <no-comparator/>
-        </suppressions>
-        <typeConfigs>
-          <no-comparator/>
-          <entry>
-            <string>checkstyle</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>checkstyle</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>codenarc</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>codenarc</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>cpd</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>cpd</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>cpplint</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>cpplint</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>csslint</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>csslint</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>findbugs</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>findbugs</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>fxcop</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>fxcop</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>gendarme</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>gendarme</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>jcreport</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>jcreport</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>jslint</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>jslint</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>pep8</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>pep8</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>perlcritic</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>perlcritic</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>pmd</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>pmd</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>pylint</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>pylint</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern>**/pylintoutput</pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>simian</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>simian</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-          <entry>
-            <string>stylecop</string>
-            <hudson.plugins.violations.TypeConfig>
-              <type>stylecop</type>
-              <min>10</min>
-              <max>999</max>
-              <unstable>999</unstable>
-              <usePattern>false</usePattern>
-              <pattern></pattern>
-            </hudson.plugins.violations.TypeConfig>
-          </entry>
-        </typeConfigs>
-        <limit>100</limit>
-        <sourcePathPattern></sourcePathPattern>
-        <fauxProjectPath></fauxProjectPath>
-        <encoding>default</encoding>
-      </config>
-    </hudson.plugins.violations.ViolationsPublisher>
-  </publishers>
-  <buildWrappers>
-    <EnvInjectBuildWrapper plugin="envinject@1.73">
-      <info>
-        <propertiesContent>PATH=/opt/local/bin:/opt/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin</propertiesContent>
-        <loadFilesFromMaster>false</loadFilesFromMaster>
-      </info>
-    </EnvInjectBuildWrapper>
-  </buildWrappers>
-  <executionStrategy class="hudson.matrix.DefaultMatrixExecutionStrategyImpl">
-    <runSequentially>false</runSequentially>
-  </executionStrategy>
-</matrix-project>
diff --git a/jenkins.sh b/jenkins.sh
deleted file mode 100755
index 3c9bdbadc..000000000
--- a/jenkins.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-cd `dirname $0`
-
-# Download nltk python dependencies
-pip install --upgrade -r pip-req.txt
-pip install --upgrade matplotlib
-pip install --upgrade https://github.com/PyCQA/pylint/archive/master.zip
-
-# Download nltk data packages
-python -c "import nltk; nltk.download('all')" || echo "NLTK data download failed: $?"
-
-# Download external dependencies
-pushd ${HOME}
-[[ ! -d 'third' ]] && mkdir 'third'
-pushd 'third'
-
-# Download SENNA
-senna_file_name=$(curl -s 'https://ronan.collobert.com/senna/download.html' | grep -o 'senna-v.*.tgz' | head -n1)
-senna_folder_name='senna'
-if [[ ! -d $senna_folder_name ]]; then
-        wget -nv "https://ronan.collobert.com/senna/$senna_file_name"
-        tar -xvzf ${senna_file_name}
-        rm ${senna_file_name}
-fi
-
-# Setup the Environment variable
-export SENNA=$(pwd)'/senna'
-
-popd
-popd
-
-echo "---- CLASSPATH: ----"
-echo $CLASSPATH
-echo "---- MODELS: ----"
-echo $STANFORD_MODELS
-echo "---- Running tests ----"
-
-# Coverage
-rm -f coverage_scrubbed.xml
-pytest --cov=nltk --cov-report xml nltk/test/
-iconv -c -f utf-8 -t utf-8 coverage.xml > coverage_scrubbed.xml
-
-# Create a default pylint configuration file.
-touch ~/.pylintrc
-pylint -f parseable nltk > pylintoutput
-
-# Script always succeeds
-true
diff --git a/nltk/__init__.py b/nltk/__init__.py
index 1cd72fb45..a96ac22b2 100644
--- a/nltk/__init__.py
+++ b/nltk/__init__.py
@@ -135,7 +135,6 @@ from nltk.featstruct import *
 from nltk.grammar import *
 from nltk.probability import *
 from nltk.text import *
-from nltk.tree import *
 from nltk.util import *
 from nltk.jsontags import *
 
@@ -151,6 +150,7 @@ from nltk.parse import *
 from nltk.tag import *
 from nltk.tokenize import *
 from nltk.translate import *
+from nltk.tree import *
 from nltk.sem import *
 from nltk.stem import *
 
@@ -200,7 +200,7 @@ else:
 from nltk import ccg, chunk, classify, collocations
 from nltk import data, featstruct, grammar, help, inference, metrics
 from nltk import misc, parse, probability, sem, stem, wsd
-from nltk import tag, tbl, text, tokenize, translate, tree, treetransforms, util
+from nltk import tag, tbl, text, tokenize, translate, tree, util
 
 
 # FIXME:  override any accidentally imported demo, see https://github.com/nltk/nltk/issues/2116
diff --git a/nltk/grammar.py b/nltk/grammar.py
index 49c38300a..630d5c6f7 100644
--- a/nltk/grammar.py
+++ b/nltk/grammar.py
@@ -5,6 +5,7 @@
 #         Edward Loper <edloper@gmail.com>
 #         Jason Narad <jason.narad@gmail.com>
 #         Peter Ljunglöf <peter.ljunglof@heatherleaf.se>
+#         Tom Aarsen <>
 # URL: <https://www.nltk.org/>
 # For license information, see LICENSE.TXT
 #
@@ -1317,7 +1318,7 @@ def _read_fcfg_production(input, fstruct_reader):
 
 _ARROW_RE = re.compile(r"\s* -> \s*", re.VERBOSE)
 _PROBABILITY_RE = re.compile(r"( \[ [\d\.]+ \] ) \s*", re.VERBOSE)
-_TERMINAL_RE = re.compile(r'( "[^"]+" | \'[^\']+\' ) \s*', re.VERBOSE)
+_TERMINAL_RE = re.compile(r'( "[^"]*" | \'[^\']*\' ) \s*', re.VERBOSE)
 _DISJUNCTION_RE = re.compile(r"\| \s*", re.VERBOSE)
 
 
@@ -1531,48 +1532,6 @@ def cfg_demo():
     print()
 
 
-toy_pcfg1 = PCFG.fromstring(
-    """
-    S -> NP VP [1.0]
-    NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
-    Det -> 'the' [0.8] | 'my' [0.2]
-    N -> 'man' [0.5] | 'telescope' [0.5]
-    VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
-    V -> 'ate' [0.35] | 'saw' [0.65]
-    PP -> P NP [1.0]
-    P -> 'with' [0.61] | 'under' [0.39]
-    """
-)
-
-toy_pcfg2 = PCFG.fromstring(
-    """
-    S    -> NP VP         [1.0]
-    VP   -> V NP          [.59]
-    VP   -> V             [.40]
-    VP   -> VP PP         [.01]
-    NP   -> Det N         [.41]
-    NP   -> Name          [.28]
-    NP   -> NP PP         [.31]
-    PP   -> P NP          [1.0]
-    V    -> 'saw'         [.21]
-    V    -> 'ate'         [.51]
-    V    -> 'ran'         [.28]
-    N    -> 'boy'         [.11]
-    N    -> 'cookie'      [.12]
-    N    -> 'table'       [.13]
-    N    -> 'telescope'   [.14]
-    N    -> 'hill'        [.5]
-    Name -> 'Jack'        [.52]
-    Name -> 'Bob'         [.48]
-    P    -> 'with'        [.61]
-    P    -> 'under'       [.39]
-    Det  -> 'the'         [.41]
-    Det  -> 'a'           [.31]
-    Det  -> 'my'          [.28]
-    """
-)
-
-
 def pcfg_demo():
     """
     A demonstration showing how a ``PCFG`` can be created and used.
@@ -1582,6 +1541,47 @@ def pcfg_demo():
     from nltk.corpus import treebank
     from nltk.parse import pchart
 
+    toy_pcfg1 = PCFG.fromstring(
+        """
+        S -> NP VP [1.0]
+        NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
+        Det -> 'the' [0.8] | 'my' [0.2]
+        N -> 'man' [0.5] | 'telescope' [0.5]
+        VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
+        V -> 'ate' [0.35] | 'saw' [0.65]
+        PP -> P NP [1.0]
+        P -> 'with' [0.61] | 'under' [0.39]
+        """
+    )
+
+    toy_pcfg2 = PCFG.fromstring(
+        """
+        S    -> NP VP         [1.0]
+        VP   -> V NP          [.59]
+        VP   -> V             [.40]
+        VP   -> VP PP         [.01]
+        NP   -> Det N         [.41]
+        NP   -> Name          [.28]
+        NP   -> NP PP         [.31]
+        PP   -> P NP          [1.0]
+        V    -> 'saw'         [.21]
+        V    -> 'ate'         [.51]
+        V    -> 'ran'         [.28]
+        N    -> 'boy'         [.11]
+        N    -> 'cookie'      [.12]
+        N    -> 'table'       [.13]
+        N    -> 'telescope'   [.14]
+        N    -> 'hill'        [.5]
+        Name -> 'Jack'        [.52]
+        Name -> 'Bob'         [.48]
+        P    -> 'with'        [.61]
+        P    -> 'under'       [.39]
+        Det  -> 'the'         [.41]
+        Det  -> 'a'           [.31]
+        Det  -> 'my'          [.28]
+        """
+    )
+
     pcfg_prods = toy_pcfg1.productions()
 
     pcfg_prod = pcfg_prods[2]
diff --git a/nltk/parse/generate.py b/nltk/parse/generate.py
index 951020afa..9eaa92584 100644
--- a/nltk/parse/generate.py
+++ b/nltk/parse/generate.py
@@ -42,14 +42,11 @@ def _generate_all(grammar, items, depth):
             for frag1 in _generate_one(grammar, items[0], depth):
                 for frag2 in _generate_all(grammar, items[1:], depth):
                     yield frag1 + frag2
-        except RuntimeError as _error:
-            if _error.message == "maximum recursion depth exceeded":
-                # Helpful error message while still showing the recursion stack.
-                raise RuntimeError(
-                    "The grammar has rule(s) that yield infinite recursion!!"
-                ) from _error
-            else:
-                raise
+        except RecursionError as error:
+            # Helpful error message while still showing the recursion stack.
+            raise RuntimeError(
+                "The grammar has rule(s) that yield infinite recursion!"
+            ) from error
     else:
         yield []
 
diff --git a/nltk/parse/util.py b/nltk/parse/util.py
index b2714e0ec..66e62d303 100644
--- a/nltk/parse/util.py
+++ b/nltk/parse/util.py
@@ -1,6 +1,7 @@
 # Natural Language Toolkit: Parser Utility Functions
 #
 # Author: Ewan Klein <ewan@inf.ed.ac.uk>
+#         Tom Aarsen <>
 #
 # Copyright (C) 2001-2021 NLTK Project
 # URL: <https://www.nltk.org/>
diff --git a/nltk/parse/viterbi.py b/nltk/parse/viterbi.py
index 3629292ef..c02765285 100644
--- a/nltk/parse/viterbi.py
+++ b/nltk/parse/viterbi.py
@@ -337,9 +337,50 @@ def demo():
     import time
 
     from nltk import tokenize
-    from nltk.grammar import toy_pcfg1, toy_pcfg2
+    from nltk.grammar import PCFG
     from nltk.parse import ViterbiParser
 
+    toy_pcfg1 = PCFG.fromstring(
+        """
+    S -> NP VP [1.0]
+    NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
+    Det -> 'the' [0.8] | 'my' [0.2]
+    N -> 'man' [0.5] | 'telescope' [0.5]
+    VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
+    V -> 'ate' [0.35] | 'saw' [0.65]
+    PP -> P NP [1.0]
+    P -> 'with' [0.61] | 'under' [0.39]
+    """
+    )
+
+    toy_pcfg2 = PCFG.fromstring(
+        """
+    S    -> NP VP         [1.0]
+    VP   -> V NP          [.59]
+    VP   -> V             [.40]
+    VP   -> VP PP         [.01]
+    NP   -> Det N         [.41]
+    NP   -> Name          [.28]
+    NP   -> NP PP         [.31]
+    PP   -> P NP          [1.0]
+    V    -> 'saw'         [.21]
+    V    -> 'ate'         [.51]
+    V    -> 'ran'         [.28]
+    N    -> 'boy'         [.11]
+    N    -> 'cookie'      [.12]
+    N    -> 'table'       [.13]
+    N    -> 'telescope'   [.14]
+    N    -> 'hill'        [.5]
+    Name -> 'Jack'        [.52]
+    Name -> 'Bob'         [.48]
+    P    -> 'with'        [.61]
+    P    -> 'under'       [.39]
+    Det  -> 'the'         [.41]
+    Det  -> 'a'           [.31]
+    Det  -> 'my'          [.28]
+    """
+    )
+
     # Define two demos.  Each demo has a sentence and a grammar.
     demos = [
         ("I saw the man with my telescope", toy_pcfg1),
diff --git a/nltk/tokenize/api.py b/nltk/tokenize/api.py
index 6786dad2b..a45c68d0b 100644
--- a/nltk/tokenize/api.py
+++ b/nltk/tokenize/api.py
@@ -11,6 +11,7 @@ Tokenizer Interface
 """
 
 from abc import ABC, abstractmethod
+from typing import Iterator, List, Tuple
 
 from nltk.internals import overridden
 from nltk.tokenize.util import string_span_tokenize
@@ -23,41 +24,43 @@ class TokenizerI(ABC):
     """
 
     @abstractmethod
-    def tokenize(self, s):
+    def tokenize(self, s: str) -> List[str]:
         """
         Return a tokenized copy of *s*.
 
-        :rtype: list of str
+        :rtype: List[str]
         """
         if overridden(self.tokenize_sents):
             return self.tokenize_sents([s])[0]
 
-    def span_tokenize(self, s):
+    def span_tokenize(self, s: str) -> Iterator[Tuple[int, int]]:
         """
         Identify the tokens using integer offsets ``(start_i, end_i)``,
         where ``s[start_i:end_i]`` is the corresponding token.
 
-        :rtype: iter(tuple(int, int))
+        :rtype: Iterator[Tuple[int, int]]
         """
         raise NotImplementedError()
 
-    def tokenize_sents(self, strings):
+    def tokenize_sents(self, strings: List[str]) -> List[List[str]]:
         """
         Apply ``self.tokenize()`` to each element of ``strings``.  I.e.:
 
             return [self.tokenize(s) for s in strings]
 
-        :rtype: list(list(str))
+        :rtype: List[List[str]]
         """
         return [self.tokenize(s) for s in strings]
 
-    def span_tokenize_sents(self, strings):
+    def span_tokenize_sents(
+        self, strings: List[str]
+    ) -> Iterator[List[Tuple[int, int]]]:
         """
         Apply ``self.span_tokenize()`` to each element of ``strings``.  I.e.:
 
             return [self.span_tokenize(s) for s in strings]
 
-        :rtype: iter(list(tuple(int, int)))
+        :yield: List[Tuple[int, int]]
         """
         for s in strings:
             yield list(self.span_tokenize(s))
diff --git a/nltk/tokenize/casual.py b/nltk/tokenize/casual.py
index 89cc63a10..c3c56c4a3 100644
--- a/nltk/tokenize/casual.py
+++ b/nltk/tokenize/casual.py
@@ -28,16 +28,16 @@ domains and tasks. The basic logic is this:
 
 4. When instantiating Tokenizer objects, there are several options:
     * preserve_case. By default, it is set to True. If it is set to
-        False, then the tokenizer will downcase everything except for
-        emoticons.
+      False, then the tokenizer will downcase everything except for
+      emoticons.
     * reduce_len. By default, it is set to False. It specifies whether
-        to replace repeated character sequences of length 3 or greater
-        with sequences of length 3.
+      to replace repeated character sequences of length 3 or greater
+      with sequences of length 3.
     * strip_handles. By default, it is set to False. It specifies
-        whether to remove Twitter handles of text used in the
-        `tokenize` method.
+      whether to remove Twitter handles of text used in the
+      `tokenize` method.
     * match_phone_numbers. By default, it is set to True. It indicates
-        whether the `tokenize` method should look for phone numbers.
+      whether the `tokenize` method should look for phone numbers.
 """
 
 
@@ -48,6 +48,8 @@ from typing import List
 
 import regex  # https://github.com/nltk/nltk/issues/2409
 
+from nltk.tokenize.api import TokenizerI
+
 ######################################################################
 # The following strings are components in the regular expression
 # that is used for tokenizing. It's important that phone_number
@@ -276,7 +278,7 @@ def _replace_html_entities(text, keep=(), remove_illegal=True, encoding="utf-8")
 ######################################################################
 
 
-class TweetTokenizer:
+class TweetTokenizer(TokenizerI):
     r"""
     Tokenizer for tweets.
 
diff --git a/nltk/tokenize/destructive.py b/nltk/tokenize/destructive.py
index 32eb64fd8..35bda4e1f 100644
--- a/nltk/tokenize/destructive.py
+++ b/nltk/tokenize/destructive.py
@@ -8,6 +8,8 @@
 
 
 import re
+import warnings
+from typing import Iterator, List, Tuple
 
 from nltk.tokenize.api import TokenizerI
 from nltk.tokenize.util import align_tokens
@@ -36,6 +38,9 @@ class NLTKWordTokenizer(TokenizerI):
     """
     The NLTK tokenizer that has improved upon the TreebankWordTokenizer.
 
+    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
+    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
+
     The tokenizer is "destructive" such that the regexes applied will munge the
     input string to a state beyond re-construction. It is possible to apply
     `TreebankWordDetokenizer.detokenize` to the tokenized outputs of
@@ -112,7 +117,43 @@ class NLTKWordTokenizer(TokenizerI):
     CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
     CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
 
-    def tokenize(self, text, convert_parentheses=False, return_str=False):
+    def tokenize(
+        self, text: str, convert_parentheses: bool = False, return_str: bool = False
+    ) -> List[str]:
+        r"""Return a tokenized copy of `text`.
+
+        >>> from nltk.tokenize import NLTKWordTokenizer
+        >>> s = '''Good muffins cost $3.88 (roughly 3,36 euros)\nin New York.  Please buy me\ntwo of them.\nThanks.'''
+        >>> NLTKWordTokenizer().tokenize(s)
+        ['Good', 'muffins', 'cost', '$', '3.88', '(', 'roughly', '3,36',
+        'euros', ')', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+        >>> NLTKWordTokenizer().tokenize(s, convert_parentheses=True)
+        ['Good', 'muffins', 'cost', '$', '3.88', '-LRB-', 'roughly', '3,36',
+        'euros', '-RRB-', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+        >>> NLTKWordTokenizer().tokenize(s, return_str=True)
+        ' Good muffins cost  $ 3.88  ( roughly 3,36 euros ) \nin New York.  Please buy me\ntwo of them.\nThanks  .  '
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :param convert_parentheses: if True, replace parentheses to PTB symbols,
+            e.g. `(` to `-LRB-`. Defaults to False.
+        :type convert_parentheses: bool, optional
+        :param return_str: If True, return tokens as space-separated string,
+            defaults to False.
+        :type return_str: bool, optional
+        :return: List of tokens from `text`.
+        :rtype: List[str]
+        """
+        if return_str:
+            warnings.warn(
+                "Parameter 'return_str' has been deprecated and should no "
+                "longer be used.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+
         for regexp, substitution in self.STARTING_QUOTES:
             text = regexp.sub(substitution, text)
 
@@ -147,11 +188,13 @@ class NLTKWordTokenizer(TokenizerI):
         # for regexp in self._contractions.CONTRACTIONS4:
         #     text = regexp.sub(r' \1 \2 \3 ', text)
 
-        return text if return_str else text.split()
+        return text.split()
 
-    def span_tokenize(self, text):
+    def span_tokenize(self, text: str) -> Iterator[Tuple[int, int]]:
         r"""
+        Returns the spans of the tokens in ``text``.
         Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
+
             >>> from nltk.tokenize import NLTKWordTokenizer
             >>> s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
             >>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
@@ -165,21 +208,10 @@ class NLTKWordTokenizer(TokenizerI):
             ... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
             >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
             True
-            Additional example
-            >>> from nltk.tokenize import NLTKWordTokenizer
-            >>> s = '''I said, "I'd like to buy some ''good muffins" which cost $3.88\n each in New (York)."'''
-            >>> expected = [(0, 1), (2, 6), (6, 7), (8, 9), (9, 10), (10, 12),
-            ... (13, 17), (18, 20), (21, 24), (25, 29), (30, 32), (32, 36),
-            ... (37, 44), (44, 45), (46, 51), (52, 56), (57, 58), (58, 62),
-            ... (64, 68), (69, 71), (72, 75), (76, 77), (77, 81), (81, 82),
-            ... (82, 83), (83, 84)]
-            >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
-            True
-            >>> expected = ['I', 'said', ',', '"', 'I', "'d", 'like', 'to',
-            ... 'buy', 'some', "''", "good", 'muffins', '"', 'which', 'cost',
-            ... '$', '3.88', 'each', 'in', 'New', '(', 'York', ')', '.', '"']
-            >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
-            True
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :yield: Tuple[int, int]
         """
         raw_tokens = self.tokenize(text)
 
diff --git a/nltk/tokenize/punkt.py b/nltk/tokenize/punkt.py
index a08ff4c90..54937b9ec 100644
--- a/nltk/tokenize/punkt.py
+++ b/nltk/tokenize/punkt.py
@@ -266,7 +266,6 @@ class PunktLanguageVars:
         return self._word_tokenizer_re().findall(s)
 
     _period_context_fmt = r"""
-        \S*                          # some word material
         %(SentEndChars)s             # a potential sentence ending
         (?=(?P<after_tok>
             %(NonWord)s              # either other punctuation
@@ -1284,8 +1283,7 @@ class PunktSentenceTokenizer(PunktBaseClass, TokenizerI):
         See format_debug_decision() to help make this output readable.
         """
 
-        for match in self._lang_vars.period_context_re().finditer(text):
-            decision_text = match.group() + match.group("after_tok")
+        for match, decision_text in self._match_potential_end_contexts(text):
             tokens = self._tokenize_words(decision_text)
             tokens = list(self._annotate_first_pass(tokens))
             while tokens and not tokens[0].tok.endswith(self._lang_vars.sent_end_chars):
@@ -1333,10 +1331,68 @@ class PunktSentenceTokenizer(PunktBaseClass, TokenizerI):
         """
         return [text[s:e] for s, e in self.span_tokenize(text, realign_boundaries)]
 
+    def _match_potential_end_contexts(self, text):
+        """
+        Given a text, find the matches of potential sentence breaks,
+        alongside the contexts surrounding these sentence breaks.
+
+        Since the fix for the ReDOS discovered in issue #2866, we no longer match
+        the word before a potential end of sentence token. Instead, we use a separate
+        regex for this. As a consequence, `finditer`'s desire to find non-overlapping
+        matches no longer aids us in finding the single longest match.
+        Where previously, we could use::
+
+            >>> pst = PunktSentenceTokenizer()
+            >>> text = "Very bad acting!!! I promise."
+            >>> list(pst._lang_vars.period_context_re().finditer(text)) # doctest: +SKIP
+            [<re.Match object; span=(9, 18), match='acting!!!'>]
+
+        Now we have to find the word before (i.e. 'acting') separately, and `finditer`
+        returns::
+
+            >>> pst = PunktSentenceTokenizer()
+            >>> text = "Very bad acting!!! I promise."
+            >>> list(pst._lang_vars.period_context_re().finditer(text)) # doctest: +NORMALIZE_WHITESPACE
+            [<re.Match object; span=(15, 16), match='!'>,
+            <re.Match object; span=(16, 17), match='!'>,
+            <re.Match object; span=(17, 18), match='!'>]
+
+        So, we need to find the word before the match from right to left, and then manually remove
+        the overlaps. That is what this method does::
+
+            >>> pst = PunktSentenceTokenizer()
+            >>> text = "Very bad acting!!! I promise."
+            >>> pst._match_potential_end_contexts(text)
+            [(<re.Match object; span=(17, 18), match='!'>, 'acting!!! I')]
+
+        :param text: String of one or more sentences
+        :type text: str
+        :return: List of match-context tuples.
+        :rtype: List[Tuple[re.Match, str]]
+        """
+        before_words = {}
+        matches = []
+        for match in reversed(list(self._lang_vars.period_context_re().finditer(text))):
+            # Ignore matches that have already been captured by matches to the right of this match
+            if matches and match.end() > before_start:
+                continue
+            # Find the word before the current match
+            split = text[: match.start()].rsplit(maxsplit=1)
+            before_start = len(split[0]) if len(split) == 2 else 0
+            before_words[match] = split[-1]
+            matches.append(match)
+
+        return [
+            (
+                match,
+                before_words[match] + match.group() + match.group("after_tok"),
+            )
+            for match in matches[::-1]
+        ]
+
     def _slices_from_text(self, text):
         last_break = 0
-        for match in self._lang_vars.period_context_re().finditer(text):
-            context = match.group() + match.group("after_tok")
+        for match, context in self._match_potential_end_contexts(text):
             if self.text_contains_sentbreak(context):
                 yield slice(last_break, match.end())
                 if match.group("next_tok"):
diff --git a/nltk/tokenize/treebank.py b/nltk/tokenize/treebank.py
index 51c202013..80ce29a91 100644
--- a/nltk/tokenize/treebank.py
+++ b/nltk/tokenize/treebank.py
@@ -18,6 +18,8 @@ and available at http://www.cis.upenn.edu/~treebank/tokenizer.sed.
 """
 
 import re
+import warnings
+from typing import Iterator, List, Tuple
 
 from nltk.tokenize.api import TokenizerI
 from nltk.tokenize.destructive import MacIntyreContractions
@@ -27,8 +29,6 @@ from nltk.tokenize.util import align_tokens
 class TreebankWordTokenizer(TokenizerI):
     r"""
     The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
-    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
-    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
 
     This tokenizer performs the following steps:
 
@@ -98,7 +98,43 @@ class TreebankWordTokenizer(TokenizerI):
     CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
     CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
 
-    def tokenize(self, text, convert_parentheses=False, return_str=False):
+    def tokenize(
+        self, text: str, convert_parentheses: bool = False, return_str: bool = False
+    ) -> List[str]:
+        r"""Return a tokenized copy of `text`.
+
+        >>> from nltk.tokenize import TreebankWordTokenizer
+        >>> s = '''Good muffins cost $3.88 (roughly 3,36 euros)\nin New York.  Please buy me\ntwo of them.\nThanks.'''
+        >>> TreebankWordTokenizer().tokenize(s)
+        ['Good', 'muffins', 'cost', '$', '3.88', '(', 'roughly', '3,36',
+        'euros', ')', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+        >>> TreebankWordTokenizer().tokenize(s, convert_parentheses=True)
+        ['Good', 'muffins', 'cost', '$', '3.88', '-LRB-', 'roughly', '3,36',
+        'euros', '-RRB-', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+        >>> TreebankWordTokenizer().tokenize(s, return_str=True)
+        ' Good muffins cost  $ 3.88  ( roughly 3,36 euros ) \nin New York.  Please buy me\ntwo of them.\nThanks .  '
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :param convert_parentheses: if True, replace parentheses to PTB symbols,
+            e.g. `(` to `-LRB-`. Defaults to False.
+        :type convert_parentheses: bool, optional
+        :param return_str: If True, return tokens as space-separated string,
+            defaults to False.
+        :type return_str: bool, optional
+        :return: List of tokens from `text`.
+        :rtype: List[str]
+        """
+        if return_str is not False:
+            warnings.warn(
+                "Parameter 'return_str' has been deprecated and should no "
+                "longer be used.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+
         for regexp, substitution in self.STARTING_QUOTES:
             text = regexp.sub(substitution, text)
 
@@ -133,10 +169,11 @@ class TreebankWordTokenizer(TokenizerI):
         # for regexp in self._contractions.CONTRACTIONS4:
         #     text = regexp.sub(r' \1 \2 \3 ', text)
 
-        return text if return_str else text.split()
+        return text.split()
 
-    def span_tokenize(self, text):
+    def span_tokenize(self, text: str) -> Iterator[Tuple[int, int]]:
         r"""
+        Returns the spans of the tokens in ``text``.
         Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
 
             >>> from nltk.tokenize import TreebankWordTokenizer
@@ -153,22 +190,9 @@ class TreebankWordTokenizer(TokenizerI):
             >>> [s[start:end] for start, end in TreebankWordTokenizer().span_tokenize(s)] == expected
             True
 
-            Additional example
-            >>> from nltk.tokenize import TreebankWordTokenizer
-            >>> s = '''I said, "I'd like to buy some ''good muffins" which cost $3.88\n each in New (York)."'''
-            >>> expected = [(0, 1), (2, 6), (6, 7), (8, 9), (9, 10), (10, 12),
-            ... (13, 17), (18, 20), (21, 24), (25, 29), (30, 32), (32, 36),
-            ... (37, 44), (44, 45), (46, 51), (52, 56), (57, 58), (58, 62),
-            ... (64, 68), (69, 71), (72, 75), (76, 77), (77, 81), (81, 82),
-            ... (82, 83), (83, 84)]
-            >>> list(TreebankWordTokenizer().span_tokenize(s)) == expected
-            True
-            >>> expected = ['I', 'said', ',', '"', 'I', "'d", 'like', 'to',
-            ... 'buy', 'some', "''", "good", 'muffins', '"', 'which', 'cost',
-            ... '$', '3.88', 'each', 'in', 'New', '(', 'York', ')', '.', '"']
-            >>> [s[start:end] for start, end in TreebankWordTokenizer().span_tokenize(s)] == expected
-            True
-
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :yield: Tuple[int, int]
         """
         raw_tokens = self.tokenize(text)
 
@@ -198,13 +222,13 @@ class TreebankWordDetokenizer(TokenizerI):
 
     Note:
 
-    - There're additional assumption mades when undoing the padding of `[;@#$%&]`
+    - There're additional assumption mades when undoing the padding of ``[;@#$%&]``
       punctuation symbols that isn't presupposed in the TreebankTokenizer.
     - There're additional regexes added in reversing the parentheses tokenization,
-       - the `r'([\]\)\}\>])\s([:;,.])'` removes the additional right padding added
-         to the closing parentheses precedding `[:;,.]`.
+       such as the ``r'([\]\)\}\>])\s([:;,.])'``, which removes the additional right
+       padding added to the closing parentheses precedding ``[:;,.]``.
     - It's not possible to return the original whitespaces as they were because
-      there wasn't explicit records of where '\n', '\t' or '\s' were removed at
+      there wasn't explicit records of where `'\n'`, `'\t'` or `'\s'` were removed at
       the text.split() operation.
 
     >>> from nltk.tokenize.treebank import TreebankWordTokenizer, TreebankWordDetokenizer
@@ -215,7 +239,7 @@ class TreebankWordDetokenizer(TokenizerI):
     >>> d.detokenize(toks)
     'Good muffins cost $3.88 in New York. Please buy me two of them. Thanks.'
 
-    The MXPOST parentheses substitution can be undone using the `convert_parentheses`
+    The MXPOST parentheses substitution can be undone using the ``convert_parentheses``
     parameter:
 
     >>> s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
@@ -231,14 +255,14 @@ class TreebankWordDetokenizer(TokenizerI):
     During tokenization it's safe to add more spaces but during detokenization,
     simply undoing the padding doesn't really help.
 
-    - During tokenization, left and right pad is added to [!?], when
-      detokenizing, only left shift the [!?] is needed.
-      Thus (re.compile(r'\s([?!])'), r'\g<1>')
+    - During tokenization, left and right pad is added to ``[!?]``, when
+      detokenizing, only left shift the ``[!?]`` is needed.
+      Thus ``(re.compile(r'\s([?!])'), r'\g<1>')``.
 
-    - During tokenization [:,] are left and right padded but when detokenizing,
+    - During tokenization ``[:,]`` are left and right padded but when detokenizing,
       only left shift is necessary and we keep right pad after comma/colon
       if the string after is a non-digit.
-      Thus (re.compile(r'\s([:,])\s([^\d])'), r'\1 \2')
+      Thus ``(re.compile(r'\s([:,])\s([^\d])'), r'\1 \2')``.
 
     >>> from nltk.tokenize.treebank import TreebankWordDetokenizer
     >>> toks = ['hello', ',', 'i', 'ca', "n't", 'feel', 'my', 'feet', '!', 'Help', '!', '!']
@@ -322,13 +346,16 @@ class TreebankWordDetokenizer(TokenizerI):
         (re.compile(r"``"), r'"'),
     ]
 
-    def tokenize(self, tokens, convert_parentheses=False):
+    def tokenize(self, tokens: List[str], convert_parentheses: bool = False) -> str:
         """
         Treebank detokenizer, created by undoing the regexes from
         the TreebankWordTokenizer.tokenize.
 
         :param tokens: A list of strings, i.e. tokenized text.
-        :type tokens: list(str)
+        :type tokens: List[str]
+        :param convert_parentheses: if True, replace PTB symbols with parentheses,
+            e.g. `-LRB-` to `(`. Defaults to False.
+        :type convert_parentheses: bool, optional
         :return: str
         """
         text = " ".join(tokens)
@@ -368,6 +395,6 @@ class TreebankWordDetokenizer(TokenizerI):
 
         return text.strip()
 
-    def detokenize(self, tokens, convert_parentheses=False):
+    def detokenize(self, tokens: List[str], convert_parentheses: bool = False) -> str:
         """Duck-typing the abstract *tokenize()*."""
         return self.tokenize(tokens, convert_parentheses)
diff --git a/nltk/translate/bleu_score.py b/nltk/translate/bleu_score.py
index 36fb0704f..e684e689d 100644
--- a/nltk/translate/bleu_score.py
+++ b/nltk/translate/bleu_score.py
@@ -81,18 +81,28 @@ def sentence_bleu(
     >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS
     0.3920...
 
+    Multiple BLEU scores can be computed at once, by supplying a list of weights.
+    E.g. for computing BLEU-2, BLEU-3 *and* BLEU-4 in one computation, use:
+    >>> weights = [
+    ...     (1./2., 1./2.),
+    ...     (1./3., 1./3., 1./3.),
+    ...     (1./4., 1./4., 1./4., 1./4.)
+    ... ]
+    >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS
+    [0.7453..., 0.6240..., 0.5045...]
+
     :param references: reference sentences
     :type references: list(list(str))
     :param hypothesis: a hypothesis sentence
     :type hypothesis: list(str)
-    :param weights: weights for unigrams, bigrams, trigrams and so on
-    :type weights: list(float)
+    :param weights: weights for unigrams, bigrams, trigrams and so on (one or a list of weights)
+    :type weights: tuple(float) / list(tuple(float))
     :param smoothing_function:
     :type smoothing_function: SmoothingFunction
     :param auto_reweigh: Option to re-normalize the weights uniformly.
     :type auto_reweigh: bool
-    :return: The sentence-level BLEU score.
-    :rtype: float
+    :return: The sentence-level BLEU score. Returns a list if multiple weights were supplied.
+    :rtype: float / list(float)
     """
     return corpus_bleu(
         [references], [hypothesis], weights, smoothing_function, auto_reweigh
@@ -146,12 +156,29 @@ def corpus_bleu(
     >>> (score1 + score2) / 2 # doctest: +ELLIPSIS
     0.6223...
 
+    Custom weights may be supplied to fine-tune the BLEU score further.
+    A tuple of float weights for unigrams, bigrams, trigrams and so on can be given.
+    >>> weights = (0.1, 0.3, 0.5, 0.1)
+    >>> corpus_bleu(list_of_references, hypotheses, weights=weights) # doctest: +ELLIPSIS
+    0.5818...
+
+    This particular weight gave extra value to trigrams.
+    Furthermore, multiple weights can be given, resulting in multiple BLEU scores.
+    >>> weights = [
+    ...     (0.5, 0.5),
+    ...     (0.333, 0.333, 0.334),
+    ...     (0.25, 0.25, 0.25, 0.25),
+    ...     (0.2, 0.2, 0.2, 0.2, 0.2)
+    ... ]
+    >>> corpus_bleu(list_of_references, hypotheses, weights=weights) # doctest: +ELLIPSIS
+    [0.8242..., 0.7067..., 0.5920..., 0.4719...]
+
     :param list_of_references: a corpus of lists of reference sentences, w.r.t. hypotheses
     :type list_of_references: list(list(list(str)))
     :param hypotheses: a list of hypothesis sentences
     :type hypotheses: list(list(str))
-    :param weights: weights for unigrams, bigrams, trigrams and so on
-    :type weights: list(float)
+    :param weights: weights for unigrams, bigrams, trigrams and so on (one or a list of weights)
+    :type weights: tuple(float) / list(tuple(float))
     :param smoothing_function:
     :type smoothing_function: SmoothingFunction
     :param auto_reweigh: Option to re-normalize the weights uniformly.
@@ -169,11 +196,17 @@ def corpus_bleu(
         "The number of hypotheses and their reference(s) should be the " "same "
     )
 
+    try:
+        weights[0][0]
+    except TypeError:
+        weights = [weights]
+    max_weight_length = max(len(weight) for weight in weights)
+
     # Iterate through each hypothesis and their corresponding references.
     for references, hypothesis in zip(list_of_references, hypotheses):
         # For each order of ngram, calculate the numerator and
         # denominator for the corpus-level modified precision.
-        for i, _ in enumerate(weights, start=1):
+        for i in range(1, max_weight_length + 1):
             p_i = modified_precision(references, hypothesis, i)
             p_numerators[i] += p_i.numerator
             p_denominators[i] += p_i.denominator
@@ -187,23 +220,17 @@ def corpus_bleu(
     # Calculate corpus-level brevity penalty.
     bp = brevity_penalty(ref_lengths, hyp_lengths)
 
-    # Uniformly re-weighting based on maximum hypothesis lengths if largest
-    # order of n-grams < 4 and weights is set at default.
-    if auto_reweigh:
-        if hyp_lengths < 4 and weights == (0.25, 0.25, 0.25, 0.25):
-            weights = (1 / hyp_lengths,) * hyp_lengths
-
     # Collects the various precision values for the different ngram orders.
     p_n = [
         Fraction(p_numerators[i], p_denominators[i], _normalize=False)
-        for i, _ in enumerate(weights, start=1)
+        for i in range(1, max_weight_length + 1)
     ]
 
     # Returns 0 if there's no matching n-grams
     # We only need to check for p_numerators[1] == 0, since if there's
     # no unigrams, there won't be any higher order ngrams.
     if p_numerators[1] == 0:
-        return 0
+        return 0 if len(weights) == 1 else [0] * len(weights)
 
     # If there's no smoothing, set use method0 from SmoothinFunction class.
     if not smoothing_function:
@@ -215,9 +242,19 @@ def corpus_bleu(
     p_n = smoothing_function(
         p_n, references=references, hypothesis=hypothesis, hyp_len=hyp_lengths
     )
-    s = (w_i * math.log(p_i) for w_i, p_i in zip(weights, p_n) if p_i > 0)
-    s = bp * math.exp(math.fsum(s))
-    return s
+
+    bleu_scores = []
+    for weight in weights:
+        # Uniformly re-weighting based on maximum hypothesis lengths if largest
+        # order of n-grams < 4 and weights is set at default.
+        if auto_reweigh:
+            if hyp_lengths < 4 and weight == (0.25, 0.25, 0.25, 0.25):
+                weight = (1 / hyp_lengths,) * hyp_lengths
+
+        s = (w_i * math.log(p_i) for w_i, p_i in zip(weight, p_n) if p_i > 0)
+        s = bp * math.exp(math.fsum(s))
+        bleu_scores.append(s)
+    return bleu_scores[0] if len(weights) == 1 else bleu_scores
 
 
 def modified_precision(references, hypothesis, n):
@@ -473,7 +510,7 @@ class SmoothingFunction:
         >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method1)) # doctest: +ELLIPSIS
         0.4118...
         >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method2)) # doctest: +ELLIPSIS
-        0.4489...
+        0.4452...
         >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method3)) # doctest: +ELLIPSIS
         0.4118...
         >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method4)) # doctest: +ELLIPSIS
diff --git a/nltk/tree/__init__.py b/nltk/tree/__init__.py
new file mode 100644
index 000000000..9b052ac6d
--- /dev/null
+++ b/nltk/tree/__init__.py
@@ -0,0 +1,35 @@
+# Natural Language Toolkit: Machine Translation
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+NLTK Tree Package
+
+This package may be used for representing hierarchical language
+structures, such as syntax trees and morphological trees.
+"""
+
+# TODO: add LabelledTree (can be used for dependency trees)
+
+from nltk.tree.immutable import (
+    ImmutableMultiParentedTree,
+    ImmutableParentedTree,
+    ImmutableProbabilisticTree,
+    ImmutableTree,
+)
+from nltk.tree.parented import MultiParentedTree, ParentedTree
+from nltk.tree.parsing import bracket_parse, sinica_parse
+from nltk.tree.prettyprinter import TreePrettyPrinter
+from nltk.tree.probabilistic import ProbabilisticTree
+from nltk.tree.transforms import (
+    chomsky_normal_form,
+    collapse_unary,
+    un_chomsky_normal_form,
+)
+from nltk.tree.tree import Tree
diff --git a/nltk/tree/immutable.py b/nltk/tree/immutable.py
new file mode 100644
index 000000000..072193927
--- /dev/null
+++ b/nltk/tree/immutable.py
@@ -0,0 +1,124 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.probability import ProbabilisticMixIn
+from nltk.tree.parented import MultiParentedTree, ParentedTree
+from nltk.tree.tree import Tree
+
+
+class ImmutableTree(Tree):
+    def __init__(self, node, children=None):
+        super().__init__(node, children)
+        # Precompute our hash value.  This ensures that we're really
+        # immutable.  It also means we only have to calculate it once.
+        try:
+            self._hash = hash((self._label, tuple(self)))
+        except (TypeError, ValueError) as e:
+            raise ValueError(
+                "%s: node value and children " "must be immutable" % type(self).__name__
+            ) from e
+
+    def __setitem__(self, index, value):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __setslice__(self, i, j, value):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __delitem__(self, index):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __delslice__(self, i, j):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __iadd__(self, other):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __imul__(self, other):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def append(self, v):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def extend(self, v):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def pop(self, v=None):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def remove(self, v):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def reverse(self):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def sort(self):
+        raise ValueError("%s may not be modified" % type(self).__name__)
+
+    def __hash__(self):
+        return self._hash
+
+    def set_label(self, value):
+        """
+        Set the node label.  This will only succeed the first time the
+        node label is set, which should occur in ImmutableTree.__init__().
+        """
+        if hasattr(self, "_label"):
+            raise ValueError("%s may not be modified" % type(self).__name__)
+        self._label = value
+
+
+class ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn):
+    def __init__(self, node, children=None, **prob_kwargs):
+        ImmutableTree.__init__(self, node, children)
+        ProbabilisticMixIn.__init__(self, **prob_kwargs)
+        self._hash = hash((self._label, tuple(self), self.prob()))
+
+    # We have to patch up these methods to make them work right:
+    def _frozen_class(self):
+        return ImmutableProbabilisticTree
+
+    def __repr__(self):
+        return f"{Tree.__repr__(self)} [{self.prob()}]"
+
+    def __str__(self):
+        return f"{self.pformat(margin=60)} [{self.prob()}]"
+
+    def copy(self, deep=False):
+        if not deep:
+            return type(self)(self._label, self, prob=self.prob())
+        else:
+            return type(self).convert(self)
+
+    @classmethod
+    def convert(cls, val):
+        if isinstance(val, Tree):
+            children = [cls.convert(child) for child in val]
+            if isinstance(val, ProbabilisticMixIn):
+                return cls(val._label, children, prob=val.prob())
+            else:
+                return cls(val._label, children, prob=1.0)
+        else:
+            return val
+
+
+class ImmutableParentedTree(ImmutableTree, ParentedTree):
+    pass
+
+
+class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree):
+    pass
+
+
+__all__ = [
+    "ImmutableProbabilisticTree",
+    "ImmutableTree",
+    "ImmutableParentedTree",
+    "ImmutableMultiParentedTree",
+]
diff --git a/nltk/tree/parented.py b/nltk/tree/parented.py
new file mode 100644
index 000000000..31955ec0d
--- /dev/null
+++ b/nltk/tree/parented.py
@@ -0,0 +1,590 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import warnings
+from abc import ABCMeta, abstractmethod
+
+from nltk.tree.tree import Tree
+from nltk.util import slice_bounds
+
+
+######################################################################
+## Parented trees
+######################################################################
+class AbstractParentedTree(Tree, metaclass=ABCMeta):
+    """
+    An abstract base class for a ``Tree`` that automatically maintains
+    pointers to parent nodes.  These parent pointers are updated
+    whenever any change is made to a tree's structure.  Two subclasses
+    are currently defined:
+
+      - ``ParentedTree`` is used for tree structures where each subtree
+        has at most one parent.  This class should be used in cases
+        where there is no"sharing" of subtrees.
+
+      - ``MultiParentedTree`` is used for tree structures where a
+        subtree may have zero or more parents.  This class should be
+        used in cases where subtrees may be shared.
+
+    Subclassing
+    ===========
+    The ``AbstractParentedTree`` class redefines all operations that
+    modify a tree's structure to call two methods, which are used by
+    subclasses to update parent information:
+
+      - ``_setparent()`` is called whenever a new child is added.
+      - ``_delparent()`` is called whenever a child is removed.
+    """
+
+    def __init__(self, node, children=None):
+        super().__init__(node, children)
+        # If children is None, the tree is read from node, and
+        # all parents will be set during parsing.
+        if children is not None:
+            # Otherwise we have to set the parent of the children.
+            # Iterate over self, and *not* children, because children
+            # might be an iterator.
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    self._setparent(child, i, dry_run=True)
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    self._setparent(child, i)
+
+    # ////////////////////////////////////////////////////////////
+    # Parent management
+    # ////////////////////////////////////////////////////////////
+    @abstractmethod
+    def _setparent(self, child, index, dry_run=False):
+        """
+        Update the parent pointer of ``child`` to point to ``self``.  This
+        method is only called if the type of ``child`` is ``Tree``;
+        i.e., it is not called when adding a leaf to a tree.  This method
+        is always called before the child is actually added to the
+        child list of ``self``.
+
+        :type child: Tree
+        :type index: int
+        :param index: The index of ``child`` in ``self``.
+        :raise TypeError: If ``child`` is a tree with an impropriate
+            type.  Typically, if ``child`` is a tree, then its type needs
+            to match the type of ``self``.  This prevents mixing of
+            different tree types (single-parented, multi-parented, and
+            non-parented).
+        :param dry_run: If true, the don't actually set the child's
+            parent pointer; just check for any error conditions, and
+            raise an exception if one is found.
+        """
+
+    @abstractmethod
+    def _delparent(self, child, index):
+        """
+        Update the parent pointer of ``child`` to not point to self.  This
+        method is only called if the type of ``child`` is ``Tree``; i.e., it
+        is not called when removing a leaf from a tree.  This method
+        is always called before the child is actually removed from the
+        child list of ``self``.
+
+        :type child: Tree
+        :type index: int
+        :param index: The index of ``child`` in ``self``.
+        """
+
+    # ////////////////////////////////////////////////////////////
+    # Methods that add/remove children
+    # ////////////////////////////////////////////////////////////
+    # Every method that adds or removes a child must make
+    # appropriate calls to _setparent() and _delparent().
+
+    def __delitem__(self, index):
+        # del ptree[start:stop]
+        if isinstance(index, slice):
+            start, stop, step = slice_bounds(self, index, allow_step=True)
+            # Clear all the children pointers.
+            for i in range(start, stop, step):
+                if isinstance(self[i], Tree):
+                    self._delparent(self[i], i)
+            # Delete the children from our child list.
+            super().__delitem__(index)
+
+        # del ptree[i]
+        elif isinstance(index, int):
+            if index < 0:
+                index += len(self)
+            if index < 0:
+                raise IndexError("index out of range")
+            # Clear the child's parent pointer.
+            if isinstance(self[index], Tree):
+                self._delparent(self[index], index)
+            # Remove the child from our child list.
+            super().__delitem__(index)
+
+        elif isinstance(index, (list, tuple)):
+            # del ptree[()]
+            if len(index) == 0:
+                raise IndexError("The tree position () may not be deleted.")
+            # del ptree[(i,)]
+            elif len(index) == 1:
+                del self[index[0]]
+            # del ptree[i1, i2, i3]
+            else:
+                del self[index[0]][index[1:]]
+
+        else:
+            raise TypeError(
+                "%s indices must be integers, not %s"
+                % (type(self).__name__, type(index).__name__)
+            )
+
+    def __setitem__(self, index, value):
+        # ptree[start:stop] = value
+        if isinstance(index, slice):
+            start, stop, step = slice_bounds(self, index, allow_step=True)
+            # make a copy of value, in case it's an iterator
+            if not isinstance(value, (list, tuple)):
+                value = list(value)
+            # Check for any error conditions, so we can avoid ending
+            # up in an inconsistent state if an error does occur.
+            for i, child in enumerate(value):
+                if isinstance(child, Tree):
+                    self._setparent(child, start + i * step, dry_run=True)
+            # clear the child pointers of all parents we're removing
+            for i in range(start, stop, step):
+                if isinstance(self[i], Tree):
+                    self._delparent(self[i], i)
+            # set the child pointers of the new children.  We do this
+            # after clearing *all* child pointers, in case we're e.g.
+            # reversing the elements in a tree.
+            for i, child in enumerate(value):
+                if isinstance(child, Tree):
+                    self._setparent(child, start + i * step)
+            # finally, update the content of the child list itself.
+            super().__setitem__(index, value)
+
+        # ptree[i] = value
+        elif isinstance(index, int):
+            if index < 0:
+                index += len(self)
+            if index < 0:
+                raise IndexError("index out of range")
+            # if the value is not changing, do nothing.
+            if value is self[index]:
+                return
+            # Set the new child's parent pointer.
+            if isinstance(value, Tree):
+                self._setparent(value, index)
+            # Remove the old child's parent pointer
+            if isinstance(self[index], Tree):
+                self._delparent(self[index], index)
+            # Update our child list.
+            super().__setitem__(index, value)
+
+        elif isinstance(index, (list, tuple)):
+            # ptree[()] = value
+            if len(index) == 0:
+                raise IndexError("The tree position () may not be assigned to.")
+            # ptree[(i,)] = value
+            elif len(index) == 1:
+                self[index[0]] = value
+            # ptree[i1, i2, i3] = value
+            else:
+                self[index[0]][index[1:]] = value
+
+        else:
+            raise TypeError(
+                "%s indices must be integers, not %s"
+                % (type(self).__name__, type(index).__name__)
+            )
+
+    def append(self, child):
+        if isinstance(child, Tree):
+            self._setparent(child, len(self))
+        super().append(child)
+
+    def extend(self, children):
+        for child in children:
+            if isinstance(child, Tree):
+                self._setparent(child, len(self))
+            super().append(child)
+
+    def insert(self, index, child):
+        # Handle negative indexes.  Note that if index < -len(self),
+        # we do *not* raise an IndexError, unlike __getitem__.  This
+        # is done for consistency with list.__getitem__ and list.index.
+        if index < 0:
+            index += len(self)
+        if index < 0:
+            index = 0
+        # Set the child's parent, and update our child list.
+        if isinstance(child, Tree):
+            self._setparent(child, index)
+        super().insert(index, child)
+
+    def pop(self, index=-1):
+        if index < 0:
+            index += len(self)
+        if index < 0:
+            raise IndexError("index out of range")
+        if isinstance(self[index], Tree):
+            self._delparent(self[index], index)
+        return super().pop(index)
+
+    # n.b.: like `list`, this is done by equality, not identity!
+    # To remove a specific child, use del ptree[i].
+    def remove(self, child):
+        index = self.index(child)
+        if isinstance(self[index], Tree):
+            self._delparent(self[index], index)
+        super().remove(child)
+
+    # We need to implement __getslice__ and friends, even though
+    # they're deprecated, because otherwise list.__getslice__ will get
+    # called (since we're subclassing from list).  Just delegate to
+    # __getitem__ etc., but use max(0, start) and max(0, stop) because
+    # because negative indices are already handled *before*
+    # __getslice__ is called; and we don't want to double-count them.
+    if hasattr(list, "__getslice__"):
+
+        def __getslice__(self, start, stop):
+            return self.__getitem__(slice(max(0, start), max(0, stop)))
+
+        def __delslice__(self, start, stop):
+            return self.__delitem__(slice(max(0, start), max(0, stop)))
+
+        def __setslice__(self, start, stop, value):
+            return self.__setitem__(slice(max(0, start), max(0, stop)), value)
+
+    def __getnewargs__(self):
+        """Method used by the pickle module when un-pickling.
+        This method provides the arguments passed to ``__new__``
+        upon un-pickling. Without this method, ParentedTree instances
+        cannot be pickled and unpickled in Python 3.7+ onwards.
+
+        :return: Tuple of arguments for ``__new__``, i.e. the label
+            and the children of this node.
+        :rtype: Tuple[Any, List[AbstractParentedTree]]
+        """
+        return (self._label, list(self))
+
+
+class ParentedTree(AbstractParentedTree):
+    """
+    A ``Tree`` that automatically maintains parent pointers for
+    single-parented trees.  The following are methods for querying
+    the structure of a parented tree: ``parent``, ``parent_index``,
+    ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``.
+
+    Each ``ParentedTree`` may have at most one parent.  In
+    particular, subtrees may not be shared.  Any attempt to reuse a
+    single ``ParentedTree`` as a child of more than one parent (or
+    as multiple children of the same parent) will cause a
+    ``ValueError`` exception to be raised.
+
+    ``ParentedTrees`` should never be used in the same tree as ``Trees``
+    or ``MultiParentedTrees``.  Mixing tree implementations may result
+    in incorrect parent pointers and in ``TypeError`` exceptions.
+    """
+
+    def __init__(self, node, children=None):
+        self._parent = None
+        """The parent of this Tree, or None if it has no parent."""
+        super().__init__(node, children)
+        if children is None:
+            # If children is None, the tree is read from node.
+            # After parsing, the parent of the immediate children
+            # will point to an intermediate tree, not self.
+            # We fix this by brute force:
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    child._parent = None
+                    self._setparent(child, i)
+
+    def _frozen_class(self):
+        from nltk.tree.immutable import ImmutableParentedTree
+
+        return ImmutableParentedTree
+
+    def copy(self, deep=False):
+        if not deep:
+            warnings.warn(
+                f"{self.__class__.__name__} objects do not support shallow copies. Defaulting to a deep copy."
+            )
+        return super().copy(deep=True)
+
+    # /////////////////////////////////////////////////////////////////
+    # Methods
+    # /////////////////////////////////////////////////////////////////
+
+    def parent(self):
+        """The parent of this tree, or None if it has no parent."""
+        return self._parent
+
+    def parent_index(self):
+        """
+        The index of this tree in its parent.  I.e.,
+        ``ptree.parent()[ptree.parent_index()] is ptree``.  Note that
+        ``ptree.parent_index()`` is not necessarily equal to
+        ``ptree.parent.index(ptree)``, since the ``index()`` method
+        returns the first child that is equal to its argument.
+        """
+        if self._parent is None:
+            return None
+        for i, child in enumerate(self._parent):
+            if child is self:
+                return i
+        assert False, "expected to find self in self._parent!"
+
+    def left_sibling(self):
+        """The left sibling of this tree, or None if it has none."""
+        parent_index = self.parent_index()
+        if self._parent and parent_index > 0:
+            return self._parent[parent_index - 1]
+        return None  # no left sibling
+
+    def right_sibling(self):
+        """The right sibling of this tree, or None if it has none."""
+        parent_index = self.parent_index()
+        if self._parent and parent_index < (len(self._parent) - 1):
+            return self._parent[parent_index + 1]
+        return None  # no right sibling
+
+    def root(self):
+        """
+        The root of this tree.  I.e., the unique ancestor of this tree
+        whose parent is None.  If ``ptree.parent()`` is None, then
+        ``ptree`` is its own root.
+        """
+        root = self
+        while root.parent() is not None:
+            root = root.parent()
+        return root
+
+    def treeposition(self):
+        """
+        The tree position of this tree, relative to the root of the
+        tree.  I.e., ``ptree.root[ptree.treeposition] is ptree``.
+        """
+        if self.parent() is None:
+            return ()
+        else:
+            return self.parent().treeposition() + (self.parent_index(),)
+
+    # /////////////////////////////////////////////////////////////////
+    # Parent Management
+    # /////////////////////////////////////////////////////////////////
+
+    def _delparent(self, child, index):
+        # Sanity checks
+        assert isinstance(child, ParentedTree)
+        assert self[index] is child
+        assert child._parent is self
+
+        # Delete child's parent pointer.
+        child._parent = None
+
+    def _setparent(self, child, index, dry_run=False):
+        # If the child's type is incorrect, then complain.
+        if not isinstance(child, ParentedTree):
+            raise TypeError("Can not insert a non-ParentedTree into a ParentedTree")
+
+        # If child already has a parent, then complain.
+        if hasattr(child, "_parent") and child._parent is not None:
+            raise ValueError("Can not insert a subtree that already has a parent.")
+
+        # Set child's parent pointer & index.
+        if not dry_run:
+            child._parent = self
+
+
+class MultiParentedTree(AbstractParentedTree):
+    """
+    A ``Tree`` that automatically maintains parent pointers for
+    multi-parented trees.  The following are methods for querying the
+    structure of a multi-parented tree: ``parents()``, ``parent_indices()``,
+    ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``.
+
+    Each ``MultiParentedTree`` may have zero or more parents.  In
+    particular, subtrees may be shared.  If a single
+    ``MultiParentedTree`` is used as multiple children of the same
+    parent, then that parent will appear multiple times in its
+    ``parents()`` method.
+
+    ``MultiParentedTrees`` should never be used in the same tree as
+    ``Trees`` or ``ParentedTrees``.  Mixing tree implementations may
+    result in incorrect parent pointers and in ``TypeError`` exceptions.
+    """
+
+    def __init__(self, node, children=None):
+        self._parents = []
+        """A list of this tree's parents.  This list should not
+           contain duplicates, even if a parent contains this tree
+           multiple times."""
+        super().__init__(node, children)
+        if children is None:
+            # If children is None, the tree is read from node.
+            # After parsing, the parent(s) of the immediate children
+            # will point to an intermediate tree, not self.
+            # We fix this by brute force:
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    child._parents = []
+                    self._setparent(child, i)
+
+    def _frozen_class(self):
+        from nltk.tree.immutable import ImmutableMultiParentedTree
+
+        return ImmutableMultiParentedTree
+
+    # /////////////////////////////////////////////////////////////////
+    # Methods
+    # /////////////////////////////////////////////////////////////////
+
+    def parents(self):
+        """
+        The set of parents of this tree.  If this tree has no parents,
+        then ``parents`` is the empty set.  To check if a tree is used
+        as multiple children of the same parent, use the
+        ``parent_indices()`` method.
+
+        :type: list(MultiParentedTree)
+        """
+        return list(self._parents)
+
+    def left_siblings(self):
+        """
+        A list of all left siblings of this tree, in any of its parent
+        trees.  A tree may be its own left sibling if it is used as
+        multiple contiguous children of the same parent.  A tree may
+        appear multiple times in this list if it is the left sibling
+        of this tree with respect to multiple parents.
+
+        :type: list(MultiParentedTree)
+        """
+        return [
+            parent[index - 1]
+            for (parent, index) in self._get_parent_indices()
+            if index > 0
+        ]
+
+    def right_siblings(self):
+        """
+        A list of all right siblings of this tree, in any of its parent
+        trees.  A tree may be its own right sibling if it is used as
+        multiple contiguous children of the same parent.  A tree may
+        appear multiple times in this list if it is the right sibling
+        of this tree with respect to multiple parents.
+
+        :type: list(MultiParentedTree)
+        """
+        return [
+            parent[index + 1]
+            for (parent, index) in self._get_parent_indices()
+            if index < (len(parent) - 1)
+        ]
+
+    def _get_parent_indices(self):
+        return [
+            (parent, index)
+            for parent in self._parents
+            for index, child in enumerate(parent)
+            if child is self
+        ]
+
+    def roots(self):
+        """
+        The set of all roots of this tree.  This set is formed by
+        tracing all possible parent paths until trees with no parents
+        are found.
+
+        :type: list(MultiParentedTree)
+        """
+        return list(self._get_roots_helper({}).values())
+
+    def _get_roots_helper(self, result):
+        if self._parents:
+            for parent in self._parents:
+                parent._get_roots_helper(result)
+        else:
+            result[id(self)] = self
+        return result
+
+    def parent_indices(self, parent):
+        """
+        Return a list of the indices where this tree occurs as a child
+        of ``parent``.  If this child does not occur as a child of
+        ``parent``, then the empty list is returned.  The following is
+        always true::
+
+          for parent_index in ptree.parent_indices(parent):
+              parent[parent_index] is ptree
+        """
+        if parent not in self._parents:
+            return []
+        else:
+            return [index for (index, child) in enumerate(parent) if child is self]
+
+    def treepositions(self, root):
+        """
+        Return a list of all tree positions that can be used to reach
+        this multi-parented tree starting from ``root``.  I.e., the
+        following is always true::
+
+          for treepos in ptree.treepositions(root):
+              root[treepos] is ptree
+        """
+        if self is root:
+            return [()]
+        else:
+            return [
+                treepos + (index,)
+                for parent in self._parents
+                for treepos in parent.treepositions(root)
+                for (index, child) in enumerate(parent)
+                if child is self
+            ]
+
+    # /////////////////////////////////////////////////////////////////
+    # Parent Management
+    # /////////////////////////////////////////////////////////////////
+
+    def _delparent(self, child, index):
+        # Sanity checks
+        assert isinstance(child, MultiParentedTree)
+        assert self[index] is child
+        assert len([p for p in child._parents if p is self]) == 1
+
+        # If the only copy of child in self is at index, then delete
+        # self from child's parent list.
+        for i, c in enumerate(self):
+            if c is child and i != index:
+                break
+        else:
+            child._parents.remove(self)
+
+    def _setparent(self, child, index, dry_run=False):
+        # If the child's type is incorrect, then complain.
+        if not isinstance(child, MultiParentedTree):
+            raise TypeError(
+                "Can not insert a non-MultiParentedTree into a MultiParentedTree"
+            )
+
+        # Add self as a parent pointer if it's not already listed.
+        if not dry_run:
+            for parent in child._parents:
+                if parent is self:
+                    break
+            else:
+                child._parents.append(self)
+
+
+__all__ = [
+    "ParentedTree",
+    "MultiParentedTree",
+]
diff --git a/nltk/tree/parsing.py b/nltk/tree/parsing.py
new file mode 100644
index 000000000..062662233
--- /dev/null
+++ b/nltk/tree/parsing.py
@@ -0,0 +1,66 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+
+from nltk.tree.tree import Tree
+
+######################################################################
+## Parsing
+######################################################################
+
+
+def bracket_parse(s):
+    """
+    Use Tree.read(s, remove_empty_top_bracketing=True) instead.
+    """
+    raise NameError("Use Tree.read(s, remove_empty_top_bracketing=True) instead.")
+
+
+def sinica_parse(s):
+    """
+    Parse a Sinica Treebank string and return a tree.  Trees are represented as nested brackettings,
+    as shown in the following example (X represents a Chinese character):
+    S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY)
+
+    :return: A tree corresponding to the string representation.
+    :rtype: Tree
+    :param s: The string to be converted
+    :type s: str
+    """
+    tokens = re.split(r"([()| ])", s)
+    for i in range(len(tokens)):
+        if tokens[i] == "(":
+            tokens[i - 1], tokens[i] = (
+                tokens[i],
+                tokens[i - 1],
+            )  # pull nonterminal inside parens
+        elif ":" in tokens[i]:
+            fields = tokens[i].split(":")
+            if len(fields) == 2:  # non-terminal
+                tokens[i] = fields[1]
+            else:
+                tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")"
+        elif tokens[i] == "|":
+            tokens[i] = ""
+
+    treebank_string = " ".join(tokens)
+    return Tree.fromstring(treebank_string, remove_empty_top_bracketing=True)
+
+
+#    s = re.sub(r'^#[^\s]*\s', '', s)  # remove leading identifier
+#    s = re.sub(r'\w+:', '', s)       # remove role tags
+
+#    return s
+
+__all__ = [
+    "bracket_parse",
+    "sinica_parse",
+]
diff --git a/nltk/tree/prettyprinter.py b/nltk/tree/prettyprinter.py
new file mode 100644
index 000000000..2ce106e05
--- /dev/null
+++ b/nltk/tree/prettyprinter.py
@@ -0,0 +1,628 @@
+# Natural Language Toolkit: ASCII visualization of NLTK trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Andreas van Cranenburgh <A.W.vanCranenburgh@uva.nl>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Pretty-printing of discontinuous trees.
+Adapted from the disco-dop project, by Andreas van Cranenburgh.
+https://github.com/andreasvc/disco-dop
+
+Interesting reference (not used for this code):
+T. Eschbach et al., Orth. Hypergraph Drawing, Journal of
+Graph Algorithms and Applications, 10(2) 141--157 (2006)149.
+https://jgaa.info/accepted/2006/EschbachGuentherBecker2006.10.2.pdf
+"""
+
+import re
+
+try:
+    from html import escape
+except ImportError:
+    from cgi import escape
+
+from collections import defaultdict
+from operator import itemgetter
+
+from nltk.tree.tree import Tree
+from nltk.util import OrderedDict
+
+ANSICOLOR = {
+    "black": 30,
+    "red": 31,
+    "green": 32,
+    "yellow": 33,
+    "blue": 34,
+    "magenta": 35,
+    "cyan": 36,
+    "white": 37,
+}
+
+
+class TreePrettyPrinter:
+    """
+    Pretty-print a tree in text format, either as ASCII or Unicode.
+    The tree can be a normal tree, or discontinuous.
+
+    ``TreePrettyPrinter(tree, sentence=None, highlight=())``
+    creates an object from which different visualizations can be created.
+
+    :param tree: a Tree object.
+    :param sentence: a list of words (strings). If `sentence` is given,
+        `tree` must contain integers as leaves, which are taken as indices
+        in `sentence`. Using this you can display a discontinuous tree.
+    :param highlight: Optionally, a sequence of Tree objects in `tree` which
+        should be highlighted. Has the effect of only applying colors to nodes
+        in this sequence (nodes should be given as Tree objects, terminals as
+        indices).
+
+    >>> from nltk.tree import Tree
+    >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))')
+    >>> print(TreePrettyPrinter(tree).text())
+    ... # doctest: +NORMALIZE_WHITESPACE
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+    """
+
+    def __init__(self, tree, sentence=None, highlight=()):
+        if sentence is None:
+            leaves = tree.leaves()
+            if (
+                leaves
+                and not any(len(a) == 0 for a in tree.subtrees())
+                and all(isinstance(a, int) for a in leaves)
+            ):
+                sentence = [str(a) for a in leaves]
+            else:
+                # this deals with empty nodes (frontier non-terminals)
+                # and multiple/mixed terminals under non-terminals.
+                tree = tree.copy(True)
+                sentence = []
+                for a in tree.subtrees():
+                    if len(a) == 0:
+                        a.append(len(sentence))
+                        sentence.append(None)
+                    elif any(not isinstance(b, Tree) for b in a):
+                        for n, b in enumerate(a):
+                            if not isinstance(b, Tree):
+                                a[n] = len(sentence)
+                                if type(b) == tuple:
+                                    b = "/".join(b)
+                                sentence.append("%s" % b)
+        self.nodes, self.coords, self.edges, self.highlight = self.nodecoords(
+            tree, sentence, highlight
+        )
+
+    def __str__(self):
+        return self.text()
+
+    def __repr__(self):
+        return "<TreePrettyPrinter with %d nodes>" % len(self.nodes)
+
+    @staticmethod
+    def nodecoords(tree, sentence, highlight):
+        """
+        Produce coordinates of nodes on a grid.
+
+        Objective:
+
+        - Produce coordinates for a non-overlapping placement of nodes and
+            horizontal lines.
+        - Order edges so that crossing edges cross a minimal number of previous
+            horizontal lines (never vertical lines).
+
+        Approach:
+
+        - bottom up level order traversal (start at terminals)
+        - at each level, identify nodes which cannot be on the same row
+        - identify nodes which cannot be in the same column
+        - place nodes into a grid at (row, column)
+        - order child-parent edges with crossing edges last
+
+        Coordinates are (row, column); the origin (0, 0) is at the top left;
+        the root node is on row 0. Coordinates do not consider the size of a
+        node (which depends on font, &c), so the width of a column of the grid
+        should be automatically determined by the element with the greatest
+        width in that column. Alternatively, the integer coordinates could be
+        converted to coordinates in which the distances between adjacent nodes
+        are non-uniform.
+
+        Produces tuple (nodes, coords, edges, highlighted) where:
+
+        - nodes[id]: Tree object for the node with this integer id
+        - coords[id]: (n, m) coordinate where to draw node with id in the grid
+        - edges[id]: parent id of node with this id (ordered dictionary)
+        - highlighted: set of ids that should be highlighted
+        """
+
+        def findcell(m, matrix, startoflevel, children):
+            """
+            Find vacant row, column index for node ``m``.
+            Iterate over current rows for this level (try lowest first)
+            and look for cell between first and last child of this node,
+            add new row to level if no free row available.
+            """
+            candidates = [a for _, a in children[m]]
+            minidx, maxidx = min(candidates), max(candidates)
+            leaves = tree[m].leaves()
+            center = scale * sum(leaves) // len(leaves)  # center of gravity
+            if minidx < maxidx and not minidx < center < maxidx:
+                center = sum(candidates) // len(candidates)
+            if max(candidates) - min(candidates) > 2 * scale:
+                center -= center % scale  # round to unscaled coordinate
+                if minidx < maxidx and not minidx < center < maxidx:
+                    center += scale
+            if ids[m] == 0:
+                startoflevel = len(matrix)
+            for rowidx in range(startoflevel, len(matrix) + 1):
+                if rowidx == len(matrix):  # need to add a new row
+                    matrix.append(
+                        [
+                            vertline if a not in (corner, None) else None
+                            for a in matrix[-1]
+                        ]
+                    )
+                row = matrix[rowidx]
+                i = j = center
+                if len(children[m]) == 1:  # place unaries directly above child
+                    return rowidx, next(iter(children[m]))[1]
+                elif all(
+                    a is None or a == vertline
+                    for a in row[min(candidates) : max(candidates) + 1]
+                ):
+                    # find free column
+                    for n in range(scale):
+                        i = j = center + n
+                        while j > minidx or i < maxidx:
+                            if i < maxidx and (
+                                matrix[rowidx][i] is None or i in candidates
+                            ):
+                                return rowidx, i
+                            elif j > minidx and (
+                                matrix[rowidx][j] is None or j in candidates
+                            ):
+                                return rowidx, j
+                            i += scale
+                            j -= scale
+            raise ValueError(
+                "could not find a free cell for:\n%s\n%s"
+                "min=%d; max=%d" % (tree[m], minidx, maxidx, dumpmatrix())
+            )
+
+        def dumpmatrix():
+            """Dump matrix contents for debugging purposes."""
+            return "\n".join(
+                "%2d: %s" % (n, " ".join(("%2r" % i)[:2] for i in row))
+                for n, row in enumerate(matrix)
+            )
+
+        leaves = tree.leaves()
+        if not all(isinstance(n, int) for n in leaves):
+            raise ValueError("All leaves must be integer indices.")
+        if len(leaves) != len(set(leaves)):
+            raise ValueError("Indices must occur at most once.")
+        if not all(0 <= n < len(sentence) for n in leaves):
+            raise ValueError(
+                "All leaves must be in the interval 0..n "
+                "with n=len(sentence)\ntokens: %d indices: "
+                "%r\nsentence: %s" % (len(sentence), tree.leaves(), sentence)
+            )
+        vertline, corner = -1, -2  # constants
+        tree = tree.copy(True)
+        for a in tree.subtrees():
+            a.sort(key=lambda n: min(n.leaves()) if isinstance(n, Tree) else n)
+        scale = 2
+        crossed = set()
+        # internal nodes and lexical nodes (no frontiers)
+        positions = tree.treepositions()
+        maxdepth = max(map(len, positions)) + 1
+        childcols = defaultdict(set)
+        matrix = [[None] * (len(sentence) * scale)]
+        nodes = {}
+        ids = {a: n for n, a in enumerate(positions)}
+        highlighted_nodes = {
+            n for a, n in ids.items() if not highlight or tree[a] in highlight
+        }
+        levels = {n: [] for n in range(maxdepth - 1)}
+        terminals = []
+        for a in positions:
+            node = tree[a]
+            if isinstance(node, Tree):
+                levels[maxdepth - node.height()].append(a)
+            else:
+                terminals.append(a)
+
+        for n in levels:
+            levels[n].sort(key=lambda n: max(tree[n].leaves()) - min(tree[n].leaves()))
+        terminals.sort()
+        positions = set(positions)
+
+        for m in terminals:
+            i = int(tree[m]) * scale
+            assert matrix[0][i] is None, (matrix[0][i], m, i)
+            matrix[0][i] = ids[m]
+            nodes[ids[m]] = sentence[tree[m]]
+            if nodes[ids[m]] is None:
+                nodes[ids[m]] = "..."
+                highlighted_nodes.discard(ids[m])
+            positions.remove(m)
+            childcols[m[:-1]].add((0, i))
+
+        # add other nodes centered on their children,
+        # if the center is already taken, back off
+        # to the left and right alternately, until an empty cell is found.
+        for n in sorted(levels, reverse=True):
+            nodesatdepth = levels[n]
+            startoflevel = len(matrix)
+            matrix.append(
+                [vertline if a not in (corner, None) else None for a in matrix[-1]]
+            )
+            for m in nodesatdepth:  # [::-1]:
+                if n < maxdepth - 1 and childcols[m]:
+                    _, pivot = min(childcols[m], key=itemgetter(1))
+                    if {
+                        a[:-1]
+                        for row in matrix[:-1]
+                        for a in row[:pivot]
+                        if isinstance(a, tuple)
+                    } & {
+                        a[:-1]
+                        for row in matrix[:-1]
+                        for a in row[pivot:]
+                        if isinstance(a, tuple)
+                    }:
+                        crossed.add(m)
+
+                rowidx, i = findcell(m, matrix, startoflevel, childcols)
+                positions.remove(m)
+
+                # block positions where children of this node branch out
+                for _, x in childcols[m]:
+                    matrix[rowidx][x] = corner
+                # assert m == () or matrix[rowidx][i] in (None, corner), (
+                #         matrix[rowidx][i], m, str(tree), ' '.join(sentence))
+                # node itself
+                matrix[rowidx][i] = ids[m]
+                nodes[ids[m]] = tree[m]
+                # add column to the set of children for its parent
+                if m != ():
+                    childcols[m[:-1]].add((rowidx, i))
+        assert len(positions) == 0
+
+        # remove unused columns, right to left
+        for m in range(scale * len(sentence) - 1, -1, -1):
+            if not any(isinstance(row[m], (Tree, int)) for row in matrix):
+                for row in matrix:
+                    del row[m]
+
+        # remove unused rows, reverse
+        matrix = [
+            row
+            for row in reversed(matrix)
+            if not all(a is None or a == vertline for a in row)
+        ]
+
+        # collect coordinates of nodes
+        coords = {}
+        for n, _ in enumerate(matrix):
+            for m, i in enumerate(matrix[n]):
+                if isinstance(i, int) and i >= 0:
+                    coords[i] = n, m
+
+        # move crossed edges last
+        positions = sorted(
+            (a for level in levels.values() for a in level),
+            key=lambda a: a[:-1] in crossed,
+        )
+
+        # collect edges from node to node
+        edges = OrderedDict()
+        for i in reversed(positions):
+            for j, _ in enumerate(tree[i]):
+                edges[ids[i + (j,)]] = ids[i]
+
+        return nodes, coords, edges, highlighted_nodes
+
+    def text(
+        self,
+        nodedist=1,
+        unicodelines=False,
+        html=False,
+        ansi=False,
+        nodecolor="blue",
+        leafcolor="red",
+        funccolor="green",
+        abbreviate=None,
+        maxwidth=16,
+    ):
+        """
+        :return: ASCII art for a discontinuous tree.
+
+        :param unicodelines: whether to use Unicode line drawing characters
+            instead of plain (7-bit) ASCII.
+        :param html: whether to wrap output in html code (default plain text).
+        :param ansi: whether to produce colors with ANSI escape sequences
+            (only effective when html==False).
+        :param leafcolor, nodecolor: specify colors of leaves and phrasal
+            nodes; effective when either html or ansi is True.
+        :param abbreviate: if True, abbreviate labels longer than 5 characters.
+            If integer, abbreviate labels longer than `abbr` characters.
+        :param maxwidth: maximum number of characters before a label starts to
+            wrap; pass None to disable.
+        """
+        if abbreviate == True:
+            abbreviate = 5
+        if unicodelines:
+            horzline = "\u2500"
+            leftcorner = "\u250c"
+            rightcorner = "\u2510"
+            vertline = " \u2502 "
+            tee = horzline + "\u252C" + horzline
+            bottom = horzline + "\u2534" + horzline
+            cross = horzline + "\u253c" + horzline
+            ellipsis = "\u2026"
+        else:
+            horzline = "_"
+            leftcorner = rightcorner = " "
+            vertline = " | "
+            tee = 3 * horzline
+            cross = bottom = "_|_"
+            ellipsis = "."
+
+        def crosscell(cur, x=vertline):
+            """Overwrite center of this cell with a vertical branch."""
+            splitl = len(cur) - len(cur) // 2 - len(x) // 2 - 1
+            lst = list(cur)
+            lst[splitl : splitl + len(x)] = list(x)
+            return "".join(lst)
+
+        result = []
+        matrix = defaultdict(dict)
+        maxnodewith = defaultdict(lambda: 3)
+        maxnodeheight = defaultdict(lambda: 1)
+        maxcol = 0
+        minchildcol = {}
+        maxchildcol = {}
+        childcols = defaultdict(set)
+        labels = {}
+        wrapre = re.compile(
+            "(.{%d,%d}\\b\\W*|.{%d})" % (maxwidth - 4, maxwidth, maxwidth)
+        )
+        # collect labels and coordinates
+        for a in self.nodes:
+            row, column = self.coords[a]
+            matrix[row][column] = a
+            maxcol = max(maxcol, column)
+            label = (
+                self.nodes[a].label()
+                if isinstance(self.nodes[a], Tree)
+                else self.nodes[a]
+            )
+            if abbreviate and len(label) > abbreviate:
+                label = label[:abbreviate] + ellipsis
+            if maxwidth and len(label) > maxwidth:
+                label = wrapre.sub(r"\1\n", label).strip()
+            label = label.split("\n")
+            maxnodeheight[row] = max(maxnodeheight[row], len(label))
+            maxnodewith[column] = max(maxnodewith[column], max(map(len, label)))
+            labels[a] = label
+            if a not in self.edges:
+                continue  # e.g., root
+            parent = self.edges[a]
+            childcols[parent].add((row, column))
+            minchildcol[parent] = min(minchildcol.get(parent, column), column)
+            maxchildcol[parent] = max(maxchildcol.get(parent, column), column)
+        # bottom up level order traversal
+        for row in sorted(matrix, reverse=True):
+            noderows = [
+                ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
+                for _ in range(maxnodeheight[row])
+            ]
+            branchrow = ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
+            for col in matrix[row]:
+                n = matrix[row][col]
+                node = self.nodes[n]
+                text = labels[n]
+                if isinstance(node, Tree):
+                    # draw horizontal branch towards children for this node
+                    if n in minchildcol and minchildcol[n] < maxchildcol[n]:
+                        i, j = minchildcol[n], maxchildcol[n]
+                        a, b = (maxnodewith[i] + 1) // 2 - 1, maxnodewith[j] // 2
+                        branchrow[i] = ((" " * a) + leftcorner).ljust(
+                            maxnodewith[i], horzline
+                        )
+                        branchrow[j] = (rightcorner + (" " * b)).rjust(
+                            maxnodewith[j], horzline
+                        )
+                        for i in range(minchildcol[n] + 1, maxchildcol[n]):
+                            if i == col and any(a == i for _, a in childcols[n]):
+                                line = cross
+                            elif i == col:
+                                line = bottom
+                            elif any(a == i for _, a in childcols[n]):
+                                line = tee
+                            else:
+                                line = horzline
+                            branchrow[i] = line.center(maxnodewith[i], horzline)
+                    else:  # if n and n in minchildcol:
+                        branchrow[col] = crosscell(branchrow[col])
+                text = [a.center(maxnodewith[col]) for a in text]
+                color = nodecolor if isinstance(node, Tree) else leafcolor
+                if isinstance(node, Tree) and node.label().startswith("-"):
+                    color = funccolor
+                if html:
+                    text = [escape(a, quote=False) for a in text]
+                    if n in self.highlight:
+                        text = [f"<font color={color}>{a}</font>" for a in text]
+                elif ansi and n in self.highlight:
+                    text = ["\x1b[%d;1m%s\x1b[0m" % (ANSICOLOR[color], a) for a in text]
+                for x in range(maxnodeheight[row]):
+                    # draw vertical lines in partially filled multiline node
+                    # labels, but only if it's not a frontier node.
+                    noderows[x][col] = (
+                        text[x]
+                        if x < len(text)
+                        else (vertline if childcols[n] else " ").center(
+                            maxnodewith[col], " "
+                        )
+                    )
+            # for each column, if there is a node below us which has a parent
+            # above us, draw a vertical branch in that column.
+            if row != max(matrix):
+                for n, (childrow, col) in self.coords.items():
+                    if n > 0 and self.coords[self.edges[n]][0] < row < childrow:
+                        branchrow[col] = crosscell(branchrow[col])
+                        if col not in matrix[row]:
+                            for noderow in noderows:
+                                noderow[col] = crosscell(noderow[col])
+                branchrow = [
+                    a + ((a[-1] if a[-1] != " " else b[0]) * nodedist)
+                    for a, b in zip(branchrow, branchrow[1:] + [" "])
+                ]
+                result.append("".join(branchrow))
+            result.extend(
+                (" " * nodedist).join(noderow) for noderow in reversed(noderows)
+            )
+        return "\n".join(reversed(result)) + "\n"
+
+    def svg(self, nodecolor="blue", leafcolor="red", funccolor="green"):
+        """
+        :return: SVG representation of a tree.
+        """
+        fontsize = 12
+        hscale = 40
+        vscale = 25
+        hstart = vstart = 20
+        width = max(col for _, col in self.coords.values())
+        height = max(row for row, _ in self.coords.values())
+        result = [
+            '<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
+            'width="%dem" height="%dem" viewBox="%d %d %d %d">'
+            % (
+                width * 3,
+                height * 2.5,
+                -hstart,
+                -vstart,
+                width * hscale + 3 * hstart,
+                height * vscale + 3 * vstart,
+            )
+        ]
+
+        children = defaultdict(set)
+        for n in self.nodes:
+            if n:
+                children[self.edges[n]].add(n)
+
+        # horizontal branches from nodes to children
+        for node in self.nodes:
+            if not children[node]:
+                continue
+            y, x = self.coords[node]
+            x *= hscale
+            y *= vscale
+            x += hstart
+            y += vstart + fontsize // 2
+            childx = [self.coords[c][1] for c in children[node]]
+            xmin = hstart + hscale * min(childx)
+            xmax = hstart + hscale * max(childx)
+            result.append(
+                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
+                'points="%g,%g %g,%g" />' % (xmin, y, xmax, y)
+            )
+            result.append(
+                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
+                'points="%g,%g %g,%g" />' % (x, y, x, y - fontsize // 3)
+            )
+
+        # vertical branches from children to parents
+        for child, parent in self.edges.items():
+            y, _ = self.coords[parent]
+            y *= vscale
+            y += vstart + fontsize // 2
+            childy, childx = self.coords[child]
+            childx *= hscale
+            childy *= vscale
+            childx += hstart
+            childy += vstart - fontsize
+            result += [
+                '\t<polyline style="stroke:white; stroke-width:10; fill:none;"'
+                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y + 5),
+                '\t<polyline style="stroke:black; stroke-width:1; fill:none;"'
+                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y),
+            ]
+
+        # write nodes with coordinates
+        for n, (row, column) in self.coords.items():
+            node = self.nodes[n]
+            x = column * hscale + hstart
+            y = row * vscale + vstart
+            if n in self.highlight:
+                color = nodecolor if isinstance(node, Tree) else leafcolor
+                if isinstance(node, Tree) and node.label().startswith("-"):
+                    color = funccolor
+            else:
+                color = "black"
+            result += [
+                '\t<text style="text-anchor: middle; fill: %s; '
+                'font-size: %dpx;" x="%g" y="%g">%s</text>'
+                % (
+                    color,
+                    fontsize,
+                    x,
+                    y,
+                    escape(
+                        node.label() if isinstance(node, Tree) else node, quote=False
+                    ),
+                )
+            ]
+
+        result += ["</svg>"]
+        return "\n".join(result)
+
+
+def test():
+    """Do some tree drawing tests."""
+
+    def print_tree(n, tree, sentence=None, ansi=True, **xargs):
+        print()
+        print('{}: "{}"'.format(n, " ".join(sentence or tree.leaves())))
+        print(tree)
+        print()
+        drawtree = TreePrettyPrinter(tree, sentence)
+        try:
+            print(drawtree.text(unicodelines=ansi, ansi=ansi, **xargs))
+        except (UnicodeDecodeError, UnicodeEncodeError):
+            print(drawtree.text(unicodelines=False, ansi=False, **xargs))
+
+    from nltk.corpus import treebank
+
+    for n in [0, 1440, 1591, 2771, 2170]:
+        tree = treebank.parsed_sents()[n]
+        print_tree(n, tree, nodedist=2, maxwidth=8)
+    print()
+    print("ASCII version:")
+    print(TreePrettyPrinter(tree).text(nodedist=2))
+
+    tree = Tree.fromstring(
+        "(top (punct 8) (smain (noun 0) (verb 1) (inf (verb 5) (inf (verb 6) "
+        "(conj (inf (pp (prep 2) (np (det 3) (noun 4))) (verb 7)) (inf (verb 9)) "
+        "(vg 10) (inf (verb 11)))))) (punct 12))",
+        read_leaf=int,
+    )
+    sentence = (
+        "Ze had met haar moeder kunnen gaan winkelen ,"
+        " zwemmen of terrassen .".split()
+    )
+    print_tree("Discontinuous tree", tree, sentence, nodedist=2)
+
+
+__all__ = ["TreePrettyPrinter"]
+
+if __name__ == "__main__":
+    test()
diff --git a/nltk/tree/probabilistic.py b/nltk/tree/probabilistic.py
new file mode 100644
index 000000000..1d7384c80
--- /dev/null
+++ b/nltk/tree/probabilistic.py
@@ -0,0 +1,78 @@
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2021 NLTK Project
+# Author: Edward Loper <edloper@gmail.com>
+#         Steven Bird <stevenbird1@gmail.com>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Tom Aarsen <>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+
+from nltk.internals import raise_unorderable_types
+from nltk.probability import ProbabilisticMixIn
+from nltk.tree.immutable import ImmutableProbabilisticTree
+from nltk.tree.tree import Tree
+
+######################################################################
+## Probabilistic trees
+######################################################################
+
+
+class ProbabilisticTree(Tree, ProbabilisticMixIn):
+    def __init__(self, node, children=None, **prob_kwargs):
+        Tree.__init__(self, node, children)
+        ProbabilisticMixIn.__init__(self, **prob_kwargs)
+
+    # We have to patch up these methods to make them work right:
+    def _frozen_class(self):
+        return ImmutableProbabilisticTree
+
+    def __repr__(self):
+        return f"{Tree.__repr__(self)} (p={self.prob()!r})"
+
+    def __str__(self):
+        return f"{self.pformat(margin=60)} (p={self.prob():.6g})"
+
+    def copy(self, deep=False):
+        if not deep:
+            return type(self)(self._label, self, prob=self.prob())
+        else:
+            return type(self).convert(self)
+
+    @classmethod
+    def convert(cls, val):
+        if isinstance(val, Tree):
+            children = [cls.convert(child) for child in val]
+            if isinstance(val, ProbabilisticMixIn):
+                return cls(val._label, children, prob=val.prob())
+            else:
+                return cls(val._label, children, prob=1.0)
+        else:
+            return val
+
+    def __eq__(self, other):
+        return (
+            self.__class__ is other.__class__
+            and (
+                self._label,
+                list(self),
+                self.prob(),
+            )
+            == (other._label, list(other), other.prob())
+        )
+
+    def __lt__(self, other):
+        if not isinstance(other, Tree):
+            raise_unorderable_types("<", self, other)
+        if self.__class__ is other.__class__:
+            return (self._label, list(self), self.prob()) < (
+                other._label,
+                list(other),
+                other.prob(),
+            )
+        else:
+            return self.__class__.__name__ < other.__class__.__name__
+
+
+__all__ = ["ProbabilisticTree"]
diff --git a/nltk/tree/transforms.py b/nltk/tree/transforms.py
new file mode 100644
index 000000000..2ebaa4dbf
--- /dev/null
+++ b/nltk/tree/transforms.py
@@ -0,0 +1,338 @@
+# Natural Language Toolkit: Tree Transformations
+#
+# Copyright (C) 2005-2007 Oregon Graduate Institute
+# Author: Nathan Bodenstab <bodenstab@cslu.ogi.edu>
+# URL: <https://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+r"""
+A collection of methods for tree (grammar) transformations used
+in parsing natural language.
+
+Although many of these methods are technically grammar transformations
+(ie. Chomsky Norm Form), when working with treebanks it is much more
+natural to visualize these modifications in a tree structure.  Hence,
+we will do all transformation directly to the tree itself.
+Transforming the tree directly also allows us to do parent annotation.
+A grammar can then be simply induced from the modified tree.
+
+The following is a short tutorial on the available transformations.
+
+ 1. Chomsky Normal Form (binarization)
+
+    It is well known that any grammar has a Chomsky Normal Form (CNF)
+    equivalent grammar where CNF is defined by every production having
+    either two non-terminals or one terminal on its right hand side.
+    When we have hierarchically structured data (ie. a treebank), it is
+    natural to view this in terms of productions where the root of every
+    subtree is the head (left hand side) of the production and all of
+    its children are the right hand side constituents.  In order to
+    convert a tree into CNF, we simply need to ensure that every subtree
+    has either two subtrees as children (binarization), or one leaf node
+    (non-terminal).  In order to binarize a subtree with more than two
+    children, we must introduce artificial nodes.
+
+    There are two popular methods to convert a tree into CNF: left
+    factoring and right factoring.  The following example demonstrates
+    the difference between them.  Example::
+
+     Original       Right-Factored     Left-Factored
+
+          A              A                      A
+        / | \          /   \                  /   \
+       B  C  D   ==>  B    A|<C-D>   OR   A|<B-C>  D
+                            /  \          /  \
+                           C    D        B    C
+
+ 2. Parent Annotation
+
+    In addition to binarizing the tree, there are two standard
+    modifications to node labels we can do in the same traversal: parent
+    annotation and Markov order-N smoothing (or sibling smoothing).
+
+    The purpose of parent annotation is to refine the probabilities of
+    productions by adding a small amount of context.  With this simple
+    addition, a CYK (inside-outside, dynamic programming chart parse)
+    can improve from 74% to 79% accuracy.  A natural generalization from
+    parent annotation is to grandparent annotation and beyond.  The
+    tradeoff becomes accuracy gain vs. computational complexity.  We
+    must also keep in mind data sparcity issues.  Example::
+
+     Original       Parent Annotation
+
+          A                A^<?>
+        / | \             /   \
+       B  C  D   ==>  B^<A>    A|<C-D>^<?>     where ? is the
+                                 /  \          parent of A
+                             C^<A>   D^<A>
+
+
+ 3. Markov order-N smoothing
+
+    Markov smoothing combats data sparcity issues as well as decreasing
+    computational requirements by limiting the number of children
+    included in artificial nodes.  In practice, most people use an order
+    2 grammar.  Example::
+
+      Original       No Smoothing       Markov order 1   Markov order 2   etc.
+
+       __A__            A                      A                A
+      / /|\ \         /   \                  /   \            /   \
+     B C D E F  ==>  B    A|<C-D-E-F>  ==>  B   A|<C>  ==>   B  A|<C-D>
+                            /   \               /   \            /   \
+                           C    ...            C    ...         C    ...
+
+
+
+    Annotation decisions can be thought about in the vertical direction
+    (parent, grandparent, etc) and the horizontal direction (number of
+    siblings to keep).  Parameters to the following functions specify
+    these values.  For more information see:
+
+    Dan Klein and Chris Manning (2003) "Accurate Unlexicalized
+    Parsing", ACL-03.  https://www.aclweb.org/anthology/P03-1054
+
+ 4. Unary Collapsing
+
+    Collapse unary productions (ie. subtrees with a single child) into a
+    new non-terminal (Tree node).  This is useful when working with
+    algorithms that do not allow unary productions, yet you do not wish
+    to lose the parent information.  Example::
+
+       A
+       |
+       B   ==>   A+B
+      / \        / \
+     C   D      C   D
+
+"""
+
+from nltk.tree.tree import Tree
+
+
+def chomsky_normal_form(
+    tree, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^"
+):
+    # assume all subtrees have homogeneous children
+    # assume all terminals have no siblings
+
+    # A semi-hack to have elegant looking code below.  As a result,
+    # any subtree with a branching factor greater than 999 will be incorrectly truncated.
+    if horzMarkov is None:
+        horzMarkov = 999
+
+    # Traverse the tree depth-first keeping a list of ancestor nodes to the root.
+    # I chose not to use the tree.treepositions() method since it requires
+    # two traversals of the tree (one to get the positions, one to iterate
+    # over them) and node access time is proportional to the height of the node.
+    # This method is 7x faster which helps when parsing 40,000 sentences.
+
+    nodeList = [(tree, [tree.label()])]
+    while nodeList != []:
+        node, parent = nodeList.pop()
+        if isinstance(node, Tree):
+
+            # parent annotation
+            parentString = ""
+            originalNode = node.label()
+            if vertMarkov != 0 and node != tree and isinstance(node[0], Tree):
+                parentString = "{}<{}>".format(parentChar, "-".join(parent))
+                node.set_label(node.label() + parentString)
+                parent = [originalNode] + parent[: vertMarkov - 1]
+
+            # add children to the agenda before we mess with them
+            for child in node:
+                nodeList.append((child, parent))
+
+            # chomsky normal form factorization
+            if len(node) > 2:
+                childNodes = [child.label() for child in node]
+                nodeCopy = node.copy()
+                node[0:] = []  # delete the children
+
+                curNode = node
+                numChildren = len(nodeCopy)
+                for i in range(1, numChildren - 1):
+                    if factor == "right":
+                        newHead = "{}{}<{}>{}".format(
+                            originalNode,
+                            childChar,
+                            "-".join(
+                                childNodes[i : min([i + horzMarkov, numChildren])]
+                            ),
+                            parentString,
+                        )  # create new head
+                        newNode = Tree(newHead, [])
+                        curNode[0:] = [nodeCopy.pop(0), newNode]
+                    else:
+                        newHead = "{}{}<{}>{}".format(
+                            originalNode,
+                            childChar,
+                            "-".join(
+                                childNodes[max([numChildren - i - horzMarkov, 0]) : -i]
+                            ),
+                            parentString,
+                        )
+                        newNode = Tree(newHead, [])
+                        curNode[0:] = [newNode, nodeCopy.pop()]
+
+                    curNode = newNode
+
+                curNode[0:] = [child for child in nodeCopy]
+
+
+def un_chomsky_normal_form(
+    tree, expandUnary=True, childChar="|", parentChar="^", unaryChar="+"
+):
+    # Traverse the tree-depth first keeping a pointer to the parent for modification purposes.
+    nodeList = [(tree, [])]
+    while nodeList != []:
+        node, parent = nodeList.pop()
+        if isinstance(node, Tree):
+            # if the node contains the 'childChar' character it means that
+            # it is an artificial node and can be removed, although we still need
+            # to move its children to its parent
+            childIndex = node.label().find(childChar)
+            if childIndex != -1:
+                nodeIndex = parent.index(node)
+                parent.remove(parent[nodeIndex])
+                # Generated node was on the left if the nodeIndex is 0 which
+                # means the grammar was left factored.  We must insert the children
+                # at the beginning of the parent's children
+                if nodeIndex == 0:
+                    parent.insert(0, node[0])
+                    parent.insert(1, node[1])
+                else:
+                    parent.extend([node[0], node[1]])
+
+                # parent is now the current node so the children of parent will be added to the agenda
+                node = parent
+            else:
+                parentIndex = node.label().find(parentChar)
+                if parentIndex != -1:
+                    # strip the node name of the parent annotation
+                    node.set_label(node.label()[:parentIndex])
+
+                # expand collapsed unary productions
+                if expandUnary == True:
+                    unaryIndex = node.label().find(unaryChar)
+                    if unaryIndex != -1:
+                        newNode = Tree(
+                            node.label()[unaryIndex + 1 :], [i for i in node]
+                        )
+                        node.set_label(node.label()[:unaryIndex])
+                        node[0:] = [newNode]
+
+            for child in node:
+                nodeList.append((child, node))
+
+
+def collapse_unary(tree, collapsePOS=False, collapseRoot=False, joinChar="+"):
+    """
+    Collapse subtrees with a single child (ie. unary productions)
+    into a new non-terminal (Tree node) joined by 'joinChar'.
+    This is useful when working with algorithms that do not allow
+    unary productions, and completely removing the unary productions
+    would require loss of useful information.  The Tree is modified
+    directly (since it is passed by reference) and no value is returned.
+
+    :param tree: The Tree to be collapsed
+    :type  tree: Tree
+    :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie.
+                        Part-of-Speech tags) since they are always unary productions
+    :type  collapsePOS: bool
+    :param collapseRoot: 'False' (default) will not modify the root production
+                         if it is unary.  For the Penn WSJ treebank corpus, this corresponds
+                         to the TOP -> productions.
+    :type collapseRoot: bool
+    :param joinChar: A string used to connect collapsed node values (default = "+")
+    :type  joinChar: str
+    """
+
+    if collapseRoot == False and isinstance(tree, Tree) and len(tree) == 1:
+        nodeList = [tree[0]]
+    else:
+        nodeList = [tree]
+
+    # depth-first traversal of tree
+    while nodeList != []:
+        node = nodeList.pop()
+        if isinstance(node, Tree):
+            if (
+                len(node) == 1
+                and isinstance(node[0], Tree)
+                and (collapsePOS == True or isinstance(node[0, 0], Tree))
+            ):
+                node.set_label(node.label() + joinChar + node[0].label())
+                node[0:] = [child for child in node[0]]
+                # since we assigned the child's children to the current node,
+                # evaluate the current node again
+                nodeList.append(node)
+            else:
+                for child in node:
+                    nodeList.append(child)
+
+
+#################################################################
+# Demonstration
+#################################################################
+
+
+def demo():
+    """
+    A demonstration showing how each tree transform can be used.
+    """
+
+    from copy import deepcopy
+
+    from nltk.draw.tree import draw_trees
+    from nltk.tree.tree import Tree
+
+    # original tree from WSJ bracketed text
+    sentence = """(TOP
+  (S
+    (S
+      (VP
+        (VBN Turned)
+        (ADVP (RB loose))
+        (PP
+          (IN in)
+          (NP
+            (NP (NNP Shane) (NNP Longman) (POS 's))
+            (NN trading)
+            (NN room)))))
+    (, ,)
+    (NP (DT the) (NN yuppie) (NNS dealers))
+    (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
+    (. .)))"""
+    t = Tree.fromstring(sentence, remove_empty_top_bracketing=True)
+
+    # collapse subtrees with only one child
+    collapsedTree = deepcopy(t)
+    collapse_unary(collapsedTree)
+
+    # convert the tree to CNF
+    cnfTree = deepcopy(collapsedTree)
+    chomsky_normal_form(cnfTree)
+
+    # convert the tree to CNF with parent annotation (one level) and horizontal smoothing of order two
+    parentTree = deepcopy(collapsedTree)
+    chomsky_normal_form(parentTree, horzMarkov=2, vertMarkov=1)
+
+    # convert the tree back to its original form (used to make CYK results comparable)
+    original = deepcopy(parentTree)
+    un_chomsky_normal_form(original)
+
+    # convert tree back to bracketed text
+    sentence2 = original.pprint()
+    print(sentence)
+    print(sentence2)
+    print("Sentences the same? ", sentence == sentence2)
+
+    draw_trees(t, collapsedTree, cnfTree, parentTree, original)
+
+
+if __name__ == "__main__":
+    demo()
+
+__all__ = ["chomsky_normal_form", "un_chomsky_normal_form", "collapse_unary"]
diff --git a/nltk/tree.py b/nltk/tree/tree.py
similarity index 52%
rename from nltk/tree.py
rename to nltk/tree/tree.py
index c9acd7d8f..6c983c4c6 100644
--- a/nltk/tree.py
+++ b/nltk/tree/tree.py
@@ -16,14 +16,8 @@ syntax trees and morphological trees.
 
 import re
 import sys
-from abc import ABCMeta, abstractmethod
 
 from nltk.grammar import Nonterminal, Production
-from nltk.internals import raise_unorderable_types
-from nltk.probability import ProbabilisticMixIn
-from nltk.util import slice_bounds
-
-# TODO: add LabelledTree (can be used for dependency trees)
 
 ######################################################################
 ## Trees
@@ -362,7 +356,7 @@ class Tree(list):
         form P -> C1 C2 ... Cn.
 
             >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
-            >>> t.productions()
+            >>> t.productions() # doctest: +NORMALIZE_WHITESPACE
             [S -> NP VP, NP -> D N, D -> 'the', N -> 'dog', VP -> V NP, V -> 'chased',
             NP -> D N, D -> 'the', N -> 'cat']
 
@@ -479,7 +473,7 @@ class Tree(list):
         :param parentChar: A string used to separate the node representation from its vertical annotation
         :type  parentChar: str
         """
-        from nltk.treetransforms import chomsky_normal_form
+        from nltk.tree.transforms import chomsky_normal_form
 
         chomsky_normal_form(self, factor, horzMarkov, vertMarkov, childChar, parentChar)
 
@@ -504,7 +498,7 @@ class Tree(list):
         :param unaryChar: A string joining two non-terminals in a unary production (default = "+")
         :type  unaryChar: str
         """
-        from nltk.treetransforms import un_chomsky_normal_form
+        from nltk.tree.transforms import un_chomsky_normal_form
 
         un_chomsky_normal_form(self, expandUnary, childChar, parentChar, unaryChar)
 
@@ -527,7 +521,7 @@ class Tree(list):
         :param joinChar: A string used to connect collapsed node values (default = "+")
         :type  joinChar: str
         """
-        from nltk.treetransforms import collapse_unary
+        from nltk.tree.transforms import collapse_unary
 
         collapse_unary(self, collapsePOS, collapseRoot, joinChar)
 
@@ -564,6 +558,8 @@ class Tree(list):
             return type(self).convert(self)
 
     def _frozen_class(self):
+        from nltk.tree.immutable import ImmutableTree
+
         return ImmutableTree
 
     def freeze(self, leaf_freezer=None):
@@ -768,9 +764,9 @@ class Tree(list):
         """
         Pretty-print this tree as ASCII or Unicode art.
         For explanation of the arguments, see the documentation for
-        `nltk.treeprettyprinter.TreePrettyPrinter`.
+        `nltk.tree.prettyprinter.TreePrettyPrinter`.
         """
-        from nltk.treeprettyprinter import TreePrettyPrinter
+        from nltk.tree.prettyprinter import TreePrettyPrinter
 
         print(TreePrettyPrinter(self, sentence, highlight).text(**kwargs), file=stream)
 
@@ -782,61 +778,10 @@ class Tree(list):
             childstr,
         )
 
-    def _repr_png_(self):
-        """
-        Draws and outputs in PNG for ipython.
-        PNG is used instead of PDF, since it can be displayed in the qt console and
-        has wider browser support.
-        """
-        import base64
-        import os
-        import subprocess
-        import tempfile
-
-        from nltk.draw.tree import tree_to_treesegment
-        from nltk.draw.util import CanvasFrame
-        from nltk.internals import find_binary
-
-        _canvas_frame = CanvasFrame()
-        widget = tree_to_treesegment(_canvas_frame.canvas(), self)
-        _canvas_frame.add_widget(widget)
-        x, y, w, h = widget.bbox()
-        # print_to_file uses scrollregion to set the width and height of the pdf.
-        _canvas_frame.canvas()["scrollregion"] = (0, 0, w, h)
-        with tempfile.NamedTemporaryFile() as file:
-            in_path = f"{file.name}.ps"
-            out_path = f"{file.name}.png"
-            _canvas_frame.print_to_file(in_path)
-            _canvas_frame.destroy_widget(widget)
-            try:
-                subprocess.call(
-                    [
-                        find_binary(
-                            "gs",
-                            binary_names=["gswin32c.exe", "gswin64c.exe"],
-                            env_vars=["PATH"],
-                            verbose=False,
-                        )
-                    ]
-                    + "-q -dEPSCrop -sDEVICE=png16m -r90 -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -dSAFER -dBATCH -dNOPAUSE -sOutputFile={} {}".format(
-                        out_path, in_path
-                    ).split()
-                )
-            except LookupError as e:
-                pre_error_message = str(
-                    "The Ghostscript executable isn't found.\n"
-                    "See https://web.mit.edu/ghostscript/www/Install.htm\n"
-                    "If you're using a Mac, you can try installing\n"
-                    "https://docs.brew.sh/Installation then `brew install ghostscript`"
-                )
-                print(pre_error_message, file=sys.stderr)
-                raise LookupError from e
+    def _repr_svg_(self):
+        from svgling import draw_tree
 
-            with open(out_path, "rb") as sr:
-                res = sr.read()
-            os.remove(in_path)
-            os.remove(out_path)
-            return base64.b64encode(res).decode()
+        return draw_tree(self)._repr_svg_()
 
     def __str__(self):
         return self.pformat()
@@ -945,718 +890,6 @@ class Tree(list):
             )
 
 
-class ImmutableTree(Tree):
-    def __init__(self, node, children=None):
-        super().__init__(node, children)
-        # Precompute our hash value.  This ensures that we're really
-        # immutable.  It also means we only have to calculate it once.
-        try:
-            self._hash = hash((self._label, tuple(self)))
-        except (TypeError, ValueError) as e:
-            raise ValueError(
-                "%s: node value and children " "must be immutable" % type(self).__name__
-            ) from e
-
-    def __setitem__(self, index, value):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __setslice__(self, i, j, value):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __delitem__(self, index):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __delslice__(self, i, j):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __iadd__(self, other):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __imul__(self, other):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def append(self, v):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def extend(self, v):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def pop(self, v=None):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def remove(self, v):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def reverse(self):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def sort(self):
-        raise ValueError("%s may not be modified" % type(self).__name__)
-
-    def __hash__(self):
-        return self._hash
-
-    def set_label(self, value):
-        """
-        Set the node label.  This will only succeed the first time the
-        node label is set, which should occur in ImmutableTree.__init__().
-        """
-        if hasattr(self, "_label"):
-            raise ValueError("%s may not be modified" % type(self).__name__)
-        self._label = value
-
-
-######################################################################
-## Parented trees
-######################################################################
-class AbstractParentedTree(Tree, metaclass=ABCMeta):
-    """
-    An abstract base class for a ``Tree`` that automatically maintains
-    pointers to parent nodes.  These parent pointers are updated
-    whenever any change is made to a tree's structure.  Two subclasses
-    are currently defined:
-
-      - ``ParentedTree`` is used for tree structures where each subtree
-        has at most one parent.  This class should be used in cases
-        where there is no"sharing" of subtrees.
-
-      - ``MultiParentedTree`` is used for tree structures where a
-        subtree may have zero or more parents.  This class should be
-        used in cases where subtrees may be shared.
-
-    Subclassing
-    ===========
-    The ``AbstractParentedTree`` class redefines all operations that
-    modify a tree's structure to call two methods, which are used by
-    subclasses to update parent information:
-
-      - ``_setparent()`` is called whenever a new child is added.
-      - ``_delparent()`` is called whenever a child is removed.
-    """
-
-    def __init__(self, node, children=None):
-        super().__init__(node, children)
-        # If children is None, the tree is read from node, and
-        # all parents will be set during parsing.
-        if children is not None:
-            # Otherwise we have to set the parent of the children.
-            # Iterate over self, and *not* children, because children
-            # might be an iterator.
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    self._setparent(child, i, dry_run=True)
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    self._setparent(child, i)
-
-    # ////////////////////////////////////////////////////////////
-    # Parent management
-    # ////////////////////////////////////////////////////////////
-    @abstractmethod
-    def _setparent(self, child, index, dry_run=False):
-        """
-        Update the parent pointer of ``child`` to point to ``self``.  This
-        method is only called if the type of ``child`` is ``Tree``;
-        i.e., it is not called when adding a leaf to a tree.  This method
-        is always called before the child is actually added to the
-        child list of ``self``.
-
-        :type child: Tree
-        :type index: int
-        :param index: The index of ``child`` in ``self``.
-        :raise TypeError: If ``child`` is a tree with an impropriate
-            type.  Typically, if ``child`` is a tree, then its type needs
-            to match the type of ``self``.  This prevents mixing of
-            different tree types (single-parented, multi-parented, and
-            non-parented).
-        :param dry_run: If true, the don't actually set the child's
-            parent pointer; just check for any error conditions, and
-            raise an exception if one is found.
-        """
-
-    @abstractmethod
-    def _delparent(self, child, index):
-        """
-        Update the parent pointer of ``child`` to not point to self.  This
-        method is only called if the type of ``child`` is ``Tree``; i.e., it
-        is not called when removing a leaf from a tree.  This method
-        is always called before the child is actually removed from the
-        child list of ``self``.
-
-        :type child: Tree
-        :type index: int
-        :param index: The index of ``child`` in ``self``.
-        """
-
-    # ////////////////////////////////////////////////////////////
-    # Methods that add/remove children
-    # ////////////////////////////////////////////////////////////
-    # Every method that adds or removes a child must make
-    # appropriate calls to _setparent() and _delparent().
-
-    def __delitem__(self, index):
-        # del ptree[start:stop]
-        if isinstance(index, slice):
-            start, stop, step = slice_bounds(self, index, allow_step=True)
-            # Clear all the children pointers.
-            for i in range(start, stop, step):
-                if isinstance(self[i], Tree):
-                    self._delparent(self[i], i)
-            # Delete the children from our child list.
-            super().__delitem__(index)
-
-        # del ptree[i]
-        elif isinstance(index, int):
-            if index < 0:
-                index += len(self)
-            if index < 0:
-                raise IndexError("index out of range")
-            # Clear the child's parent pointer.
-            if isinstance(self[index], Tree):
-                self._delparent(self[index], index)
-            # Remove the child from our child list.
-            super().__delitem__(index)
-
-        elif isinstance(index, (list, tuple)):
-            # del ptree[()]
-            if len(index) == 0:
-                raise IndexError("The tree position () may not be deleted.")
-            # del ptree[(i,)]
-            elif len(index) == 1:
-                del self[index[0]]
-            # del ptree[i1, i2, i3]
-            else:
-                del self[index[0]][index[1:]]
-
-        else:
-            raise TypeError(
-                "%s indices must be integers, not %s"
-                % (type(self).__name__, type(index).__name__)
-            )
-
-    def __setitem__(self, index, value):
-        # ptree[start:stop] = value
-        if isinstance(index, slice):
-            start, stop, step = slice_bounds(self, index, allow_step=True)
-            # make a copy of value, in case it's an iterator
-            if not isinstance(value, (list, tuple)):
-                value = list(value)
-            # Check for any error conditions, so we can avoid ending
-            # up in an inconsistent state if an error does occur.
-            for i, child in enumerate(value):
-                if isinstance(child, Tree):
-                    self._setparent(child, start + i * step, dry_run=True)
-            # clear the child pointers of all parents we're removing
-            for i in range(start, stop, step):
-                if isinstance(self[i], Tree):
-                    self._delparent(self[i], i)
-            # set the child pointers of the new children.  We do this
-            # after clearing *all* child pointers, in case we're e.g.
-            # reversing the elements in a tree.
-            for i, child in enumerate(value):
-                if isinstance(child, Tree):
-                    self._setparent(child, start + i * step)
-            # finally, update the content of the child list itself.
-            super().__setitem__(index, value)
-
-        # ptree[i] = value
-        elif isinstance(index, int):
-            if index < 0:
-                index += len(self)
-            if index < 0:
-                raise IndexError("index out of range")
-            # if the value is not changing, do nothing.
-            if value is self[index]:
-                return
-            # Set the new child's parent pointer.
-            if isinstance(value, Tree):
-                self._setparent(value, index)
-            # Remove the old child's parent pointer
-            if isinstance(self[index], Tree):
-                self._delparent(self[index], index)
-            # Update our child list.
-            super().__setitem__(index, value)
-
-        elif isinstance(index, (list, tuple)):
-            # ptree[()] = value
-            if len(index) == 0:
-                raise IndexError("The tree position () may not be assigned to.")
-            # ptree[(i,)] = value
-            elif len(index) == 1:
-                self[index[0]] = value
-            # ptree[i1, i2, i3] = value
-            else:
-                self[index[0]][index[1:]] = value
-
-        else:
-            raise TypeError(
-                "%s indices must be integers, not %s"
-                % (type(self).__name__, type(index).__name__)
-            )
-
-    def append(self, child):
-        if isinstance(child, Tree):
-            self._setparent(child, len(self))
-        super().append(child)
-
-    def extend(self, children):
-        for child in children:
-            if isinstance(child, Tree):
-                self._setparent(child, len(self))
-            super().append(child)
-
-    def insert(self, index, child):
-        # Handle negative indexes.  Note that if index < -len(self),
-        # we do *not* raise an IndexError, unlike __getitem__.  This
-        # is done for consistency with list.__getitem__ and list.index.
-        if index < 0:
-            index += len(self)
-        if index < 0:
-            index = 0
-        # Set the child's parent, and update our child list.
-        if isinstance(child, Tree):
-            self._setparent(child, index)
-        super().insert(index, child)
-
-    def pop(self, index=-1):
-        if index < 0:
-            index += len(self)
-        if index < 0:
-            raise IndexError("index out of range")
-        if isinstance(self[index], Tree):
-            self._delparent(self[index], index)
-        return super().pop(index)
-
-    # n.b.: like `list`, this is done by equality, not identity!
-    # To remove a specific child, use del ptree[i].
-    def remove(self, child):
-        index = self.index(child)
-        if isinstance(self[index], Tree):
-            self._delparent(self[index], index)
-        super().remove(child)
-
-    # We need to implement __getslice__ and friends, even though
-    # they're deprecated, because otherwise list.__getslice__ will get
-    # called (since we're subclassing from list).  Just delegate to
-    # __getitem__ etc., but use max(0, start) and max(0, stop) because
-    # because negative indices are already handled *before*
-    # __getslice__ is called; and we don't want to double-count them.
-    if hasattr(list, "__getslice__"):
-
-        def __getslice__(self, start, stop):
-            return self.__getitem__(slice(max(0, start), max(0, stop)))
-
-        def __delslice__(self, start, stop):
-            return self.__delitem__(slice(max(0, start), max(0, stop)))
-
-        def __setslice__(self, start, stop, value):
-            return self.__setitem__(slice(max(0, start), max(0, stop)), value)
-
-
-class ParentedTree(AbstractParentedTree):
-    """
-    A ``Tree`` that automatically maintains parent pointers for
-    single-parented trees.  The following are methods for querying
-    the structure of a parented tree: ``parent``, ``parent_index``,
-    ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``.
-
-    Each ``ParentedTree`` may have at most one parent.  In
-    particular, subtrees may not be shared.  Any attempt to reuse a
-    single ``ParentedTree`` as a child of more than one parent (or
-    as multiple children of the same parent) will cause a
-    ``ValueError`` exception to be raised.
-
-    ``ParentedTrees`` should never be used in the same tree as ``Trees``
-    or ``MultiParentedTrees``.  Mixing tree implementations may result
-    in incorrect parent pointers and in ``TypeError`` exceptions.
-    """
-
-    def __init__(self, node, children=None):
-        self._parent = None
-        """The parent of this Tree, or None if it has no parent."""
-        super().__init__(node, children)
-        if children is None:
-            # If children is None, the tree is read from node.
-            # After parsing, the parent of the immediate children
-            # will point to an intermediate tree, not self.
-            # We fix this by brute force:
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    child._parent = None
-                    self._setparent(child, i)
-
-    def _frozen_class(self):
-        return ImmutableParentedTree
-
-    # /////////////////////////////////////////////////////////////////
-    # Methods
-    # /////////////////////////////////////////////////////////////////
-
-    def parent(self):
-        """The parent of this tree, or None if it has no parent."""
-        return self._parent
-
-    def parent_index(self):
-        """
-        The index of this tree in its parent.  I.e.,
-        ``ptree.parent()[ptree.parent_index()] is ptree``.  Note that
-        ``ptree.parent_index()`` is not necessarily equal to
-        ``ptree.parent.index(ptree)``, since the ``index()`` method
-        returns the first child that is equal to its argument.
-        """
-        if self._parent is None:
-            return None
-        for i, child in enumerate(self._parent):
-            if child is self:
-                return i
-        assert False, "expected to find self in self._parent!"
-
-    def left_sibling(self):
-        """The left sibling of this tree, or None if it has none."""
-        parent_index = self.parent_index()
-        if self._parent and parent_index > 0:
-            return self._parent[parent_index - 1]
-        return None  # no left sibling
-
-    def right_sibling(self):
-        """The right sibling of this tree, or None if it has none."""
-        parent_index = self.parent_index()
-        if self._parent and parent_index < (len(self._parent) - 1):
-            return self._parent[parent_index + 1]
-        return None  # no right sibling
-
-    def root(self):
-        """
-        The root of this tree.  I.e., the unique ancestor of this tree
-        whose parent is None.  If ``ptree.parent()`` is None, then
-        ``ptree`` is its own root.
-        """
-        root = self
-        while root.parent() is not None:
-            root = root.parent()
-        return root
-
-    def treeposition(self):
-        """
-        The tree position of this tree, relative to the root of the
-        tree.  I.e., ``ptree.root[ptree.treeposition] is ptree``.
-        """
-        if self.parent() is None:
-            return ()
-        else:
-            return self.parent().treeposition() + (self.parent_index(),)
-
-    # /////////////////////////////////////////////////////////////////
-    # Parent Management
-    # /////////////////////////////////////////////////////////////////
-
-    def _delparent(self, child, index):
-        # Sanity checks
-        assert isinstance(child, ParentedTree)
-        assert self[index] is child
-        assert child._parent is self
-
-        # Delete child's parent pointer.
-        child._parent = None
-
-    def _setparent(self, child, index, dry_run=False):
-        # If the child's type is incorrect, then complain.
-        if not isinstance(child, ParentedTree):
-            raise TypeError(
-                "Can not insert a non-ParentedTree " + "into a ParentedTree"
-            )
-
-        # If child already has a parent, then complain.
-        if child._parent is not None:
-            raise ValueError("Can not insert a subtree that already " "has a parent.")
-
-        # Set child's parent pointer & index.
-        if not dry_run:
-            child._parent = self
-
-
-class MultiParentedTree(AbstractParentedTree):
-    """
-    A ``Tree`` that automatically maintains parent pointers for
-    multi-parented trees.  The following are methods for querying the
-    structure of a multi-parented tree: ``parents()``, ``parent_indices()``,
-    ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``.
-
-    Each ``MultiParentedTree`` may have zero or more parents.  In
-    particular, subtrees may be shared.  If a single
-    ``MultiParentedTree`` is used as multiple children of the same
-    parent, then that parent will appear multiple times in its
-    ``parents()`` method.
-
-    ``MultiParentedTrees`` should never be used in the same tree as
-    ``Trees`` or ``ParentedTrees``.  Mixing tree implementations may
-    result in incorrect parent pointers and in ``TypeError`` exceptions.
-    """
-
-    def __init__(self, node, children=None):
-        self._parents = []
-        """A list of this tree's parents.  This list should not
-           contain duplicates, even if a parent contains this tree
-           multiple times."""
-        super().__init__(node, children)
-        if children is None:
-            # If children is None, the tree is read from node.
-            # After parsing, the parent(s) of the immediate children
-            # will point to an intermediate tree, not self.
-            # We fix this by brute force:
-            for i, child in enumerate(self):
-                if isinstance(child, Tree):
-                    child._parents = []
-                    self._setparent(child, i)
-
-    def _frozen_class(self):
-        return ImmutableMultiParentedTree
-
-    # /////////////////////////////////////////////////////////////////
-    # Methods
-    # /////////////////////////////////////////////////////////////////
-
-    def parents(self):
-        """
-        The set of parents of this tree.  If this tree has no parents,
-        then ``parents`` is the empty set.  To check if a tree is used
-        as multiple children of the same parent, use the
-        ``parent_indices()`` method.
-
-        :type: list(MultiParentedTree)
-        """
-        return list(self._parents)
-
-    def left_siblings(self):
-        """
-        A list of all left siblings of this tree, in any of its parent
-        trees.  A tree may be its own left sibling if it is used as
-        multiple contiguous children of the same parent.  A tree may
-        appear multiple times in this list if it is the left sibling
-        of this tree with respect to multiple parents.
-
-        :type: list(MultiParentedTree)
-        """
-        return [
-            parent[index - 1]
-            for (parent, index) in self._get_parent_indices()
-            if index > 0
-        ]
-
-    def right_siblings(self):
-        """
-        A list of all right siblings of this tree, in any of its parent
-        trees.  A tree may be its own right sibling if it is used as
-        multiple contiguous children of the same parent.  A tree may
-        appear multiple times in this list if it is the right sibling
-        of this tree with respect to multiple parents.
-
-        :type: list(MultiParentedTree)
-        """
-        return [
-            parent[index + 1]
-            for (parent, index) in self._get_parent_indices()
-            if index < (len(parent) - 1)
-        ]
-
-    def _get_parent_indices(self):
-        return [
-            (parent, index)
-            for parent in self._parents
-            for index, child in enumerate(parent)
-            if child is self
-        ]
-
-    def roots(self):
-        """
-        The set of all roots of this tree.  This set is formed by
-        tracing all possible parent paths until trees with no parents
-        are found.
-
-        :type: list(MultiParentedTree)
-        """
-        return list(self._get_roots_helper({}).values())
-
-    def _get_roots_helper(self, result):
-        if self._parents:
-            for parent in self._parents:
-                parent._get_roots_helper(result)
-        else:
-            result[id(self)] = self
-        return result
-
-    def parent_indices(self, parent):
-        """
-        Return a list of the indices where this tree occurs as a child
-        of ``parent``.  If this child does not occur as a child of
-        ``parent``, then the empty list is returned.  The following is
-        always true::
-
-          for parent_index in ptree.parent_indices(parent):
-              parent[parent_index] is ptree
-        """
-        if parent not in self._parents:
-            return []
-        else:
-            return [index for (index, child) in enumerate(parent) if child is self]
-
-    def treepositions(self, root):
-        """
-        Return a list of all tree positions that can be used to reach
-        this multi-parented tree starting from ``root``.  I.e., the
-        following is always true::
-
-          for treepos in ptree.treepositions(root):
-              root[treepos] is ptree
-        """
-        if self is root:
-            return [()]
-        else:
-            return [
-                treepos + (index,)
-                for parent in self._parents
-                for treepos in parent.treepositions(root)
-                for (index, child) in enumerate(parent)
-                if child is self
-            ]
-
-    # /////////////////////////////////////////////////////////////////
-    # Parent Management
-    # /////////////////////////////////////////////////////////////////
-
-    def _delparent(self, child, index):
-        # Sanity checks
-        assert isinstance(child, MultiParentedTree)
-        assert self[index] is child
-        assert len([p for p in child._parents if p is self]) == 1
-
-        # If the only copy of child in self is at index, then delete
-        # self from child's parent list.
-        for i, c in enumerate(self):
-            if c is child and i != index:
-                break
-        else:
-            child._parents.remove(self)
-
-    def _setparent(self, child, index, dry_run=False):
-        # If the child's type is incorrect, then complain.
-        if not isinstance(child, MultiParentedTree):
-            raise TypeError(
-                "Can not insert a non-MultiParentedTree " + "into a MultiParentedTree"
-            )
-
-        # Add self as a parent pointer if it's not already listed.
-        if not dry_run:
-            for parent in child._parents:
-                if parent is self:
-                    break
-            else:
-                child._parents.append(self)
-
-
-class ImmutableParentedTree(ImmutableTree, ParentedTree):
-    pass
-
-
-class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree):
-    pass
-
-
-######################################################################
-## Probabilistic trees
-######################################################################
-
-
-class ProbabilisticTree(Tree, ProbabilisticMixIn):
-    def __init__(self, node, children=None, **prob_kwargs):
-        Tree.__init__(self, node, children)
-        ProbabilisticMixIn.__init__(self, **prob_kwargs)
-
-    # We have to patch up these methods to make them work right:
-    def _frozen_class(self):
-        return ImmutableProbabilisticTree
-
-    def __repr__(self):
-        return f"{Tree.__repr__(self)} (p={self.prob()!r})"
-
-    def __str__(self):
-        return f"{self.pformat(margin=60)} (p={self.prob():.6g})"
-
-    def copy(self, deep=False):
-        if not deep:
-            return type(self)(self._label, self, prob=self.prob())
-        else:
-            return type(self).convert(self)
-
-    @classmethod
-    def convert(cls, val):
-        if isinstance(val, Tree):
-            children = [cls.convert(child) for child in val]
-            if isinstance(val, ProbabilisticMixIn):
-                return cls(val._label, children, prob=val.prob())
-            else:
-                return cls(val._label, children, prob=1.0)
-        else:
-            return val
-
-    def __eq__(self, other):
-        return (
-            self.__class__ is other.__class__
-            and (
-                self._label,
-                list(self),
-                self.prob(),
-            )
-            == (other._label, list(other), other.prob())
-        )
-
-    def __lt__(self, other):
-        if not isinstance(other, Tree):
-            raise_unorderable_types("<", self, other)
-        if self.__class__ is other.__class__:
-            return (self._label, list(self), self.prob()) < (
-                other._label,
-                list(other),
-                other.prob(),
-            )
-        else:
-            return self.__class__.__name__ < other.__class__.__name__
-
-
-class ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn):
-    def __init__(self, node, children=None, **prob_kwargs):
-        ImmutableTree.__init__(self, node, children)
-        ProbabilisticMixIn.__init__(self, **prob_kwargs)
-        self._hash = hash((self._label, tuple(self), self.prob()))
-
-    # We have to patch up these methods to make them work right:
-    def _frozen_class(self):
-        return ImmutableProbabilisticTree
-
-    def __repr__(self):
-        return f"{Tree.__repr__(self)} [{self.prob()}]"
-
-    def __str__(self):
-        return f"{self.pformat(margin=60)} [{self.prob()}]"
-
-    def copy(self, deep=False):
-        if not deep:
-            return type(self)(self._label, self, prob=self.prob())
-        else:
-            return type(self).convert(self)
-
-    @classmethod
-    def convert(cls, val):
-        if isinstance(val, Tree):
-            children = [cls.convert(child) for child in val]
-            if isinstance(val, ProbabilisticMixIn):
-                return cls(val._label, children, prob=val.prob())
-            else:
-                return cls(val._label, children, prob=1.0)
-        else:
-            return val
-
-
 def _child_names(tree):
     names = []
     for child in tree:
@@ -1667,55 +900,6 @@ def _child_names(tree):
     return names
 
 
-######################################################################
-## Parsing
-######################################################################
-
-
-def bracket_parse(s):
-    """
-    Use Tree.read(s, remove_empty_top_bracketing=True) instead.
-    """
-    raise NameError("Use Tree.read(s, remove_empty_top_bracketing=True) instead.")
-
-
-def sinica_parse(s):
-    """
-    Parse a Sinica Treebank string and return a tree.  Trees are represented as nested brackettings,
-    as shown in the following example (X represents a Chinese character):
-    S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY)
-
-    :return: A tree corresponding to the string representation.
-    :rtype: Tree
-    :param s: The string to be converted
-    :type s: str
-    """
-    tokens = re.split(r"([()| ])", s)
-    for i in range(len(tokens)):
-        if tokens[i] == "(":
-            tokens[i - 1], tokens[i] = (
-                tokens[i],
-                tokens[i - 1],
-            )  # pull nonterminal inside parens
-        elif ":" in tokens[i]:
-            fields = tokens[i].split(":")
-            if len(fields) == 2:  # non-terminal
-                tokens[i] = fields[1]
-            else:
-                tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")"
-        elif tokens[i] == "|":
-            tokens[i] = ""
-
-    treebank_string = " ".join(tokens)
-    return Tree.fromstring(treebank_string, remove_empty_top_bracketing=True)
-
-
-#    s = re.sub(r'^#[^\s]*\s', '', s)  # remove leading identifier
-#    s = re.sub(r'\w+:', '', s)       # remove role tags
-
-#    return s
-
-
 ######################################################################
 ## Demonstration
 ######################################################################
@@ -1794,15 +978,5 @@ def demo():
 
 
 __all__ = [
-    "ImmutableProbabilisticTree",
-    "ImmutableTree",
-    "ProbabilisticMixIn",
-    "ProbabilisticTree",
     "Tree",
-    "bracket_parse",
-    "sinica_parse",
-    "ParentedTree",
-    "MultiParentedTree",
-    "ImmutableParentedTree",
-    "ImmutableMultiParentedTree",
 ]
diff --git a/nltk/treeprettyprinter.py b/nltk/treeprettyprinter.py
index defdf5e9d..22686d75a 100644
--- a/nltk/treeprettyprinter.py
+++ b/nltk/treeprettyprinter.py
@@ -17,612 +17,12 @@ Graph Algorithms and Applications, 10(2) 141--157 (2006)149.
 https://jgaa.info/accepted/2006/EschbachGuentherBecker2006.10.2.pdf
 """
 
-import re
+from nltk.internals import Deprecated
+from nltk.tree.prettyprinter import TreePrettyPrinter as TPP
 
-try:
-    from html import escape
-except ImportError:
-    from cgi import escape
 
-from collections import defaultdict
-from operator import itemgetter
-
-from nltk.tree import Tree
-from nltk.util import OrderedDict
-
-ANSICOLOR = {
-    "black": 30,
-    "red": 31,
-    "green": 32,
-    "yellow": 33,
-    "blue": 34,
-    "magenta": 35,
-    "cyan": 36,
-    "white": 37,
-}
-
-
-class TreePrettyPrinter:
-    """
-    Pretty-print a tree in text format, either as ASCII or Unicode.
-    The tree can be a normal tree, or discontinuous.
-
-    ``TreePrettyPrinter(tree, sentence=None, highlight=())``
-    creates an object from which different visualizations can be created.
-
-    :param tree: a Tree object.
-    :param sentence: a list of words (strings). If `sentence` is given,
-        `tree` must contain integers as leaves, which are taken as indices
-        in `sentence`. Using this you can display a discontinuous tree.
-    :param highlight: Optionally, a sequence of Tree objects in `tree` which
-        should be highlighted. Has the effect of only applying colors to nodes
-        in this sequence (nodes should be given as Tree objects, terminals as
-        indices).
-
-    >>> from nltk.tree import Tree
-    >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))')
-    >>> print(TreePrettyPrinter(tree).text())
-    ... # doctest: +NORMALIZE_WHITESPACE
-          S
-      ____|____
-     NP        VP
-     |         |
-    Mary     walks
-    """
-
-    def __init__(self, tree, sentence=None, highlight=()):
-        if sentence is None:
-            leaves = tree.leaves()
-            if (
-                leaves
-                and not any(len(a) == 0 for a in tree.subtrees())
-                and all(isinstance(a, int) for a in leaves)
-            ):
-                sentence = [str(a) for a in leaves]
-            else:
-                # this deals with empty nodes (frontier non-terminals)
-                # and multiple/mixed terminals under non-terminals.
-                tree = tree.copy(True)
-                sentence = []
-                for a in tree.subtrees():
-                    if len(a) == 0:
-                        a.append(len(sentence))
-                        sentence.append(None)
-                    elif any(not isinstance(b, Tree) for b in a):
-                        for n, b in enumerate(a):
-                            if not isinstance(b, Tree):
-                                a[n] = len(sentence)
-                                if type(b) == tuple:
-                                    b = "/".join(b)
-                                sentence.append("%s" % b)
-        self.nodes, self.coords, self.edges, self.highlight = self.nodecoords(
-            tree, sentence, highlight
-        )
-
-    def __str__(self):
-        return self.text()
-
-    def __repr__(self):
-        return "<TreePrettyPrinter with %d nodes>" % len(self.nodes)
-
-    @staticmethod
-    def nodecoords(tree, sentence, highlight):
-        """
-        Produce coordinates of nodes on a grid.
-
-        Objective:
-
-        - Produce coordinates for a non-overlapping placement of nodes and
-            horizontal lines.
-        - Order edges so that crossing edges cross a minimal number of previous
-            horizontal lines (never vertical lines).
-
-        Approach:
-
-        - bottom up level order traversal (start at terminals)
-        - at each level, identify nodes which cannot be on the same row
-        - identify nodes which cannot be in the same column
-        - place nodes into a grid at (row, column)
-        - order child-parent edges with crossing edges last
-
-        Coordinates are (row, column); the origin (0, 0) is at the top left;
-        the root node is on row 0. Coordinates do not consider the size of a
-        node (which depends on font, &c), so the width of a column of the grid
-        should be automatically determined by the element with the greatest
-        width in that column. Alternatively, the integer coordinates could be
-        converted to coordinates in which the distances between adjacent nodes
-        are non-uniform.
-
-        Produces tuple (nodes, coords, edges, highlighted) where:
-
-        - nodes[id]: Tree object for the node with this integer id
-        - coords[id]: (n, m) coordinate where to draw node with id in the grid
-        - edges[id]: parent id of node with this id (ordered dictionary)
-        - highlighted: set of ids that should be highlighted
-        """
-
-        def findcell(m, matrix, startoflevel, children):
-            """
-            Find vacant row, column index for node ``m``.
-            Iterate over current rows for this level (try lowest first)
-            and look for cell between first and last child of this node,
-            add new row to level if no free row available.
-            """
-            candidates = [a for _, a in children[m]]
-            minidx, maxidx = min(candidates), max(candidates)
-            leaves = tree[m].leaves()
-            center = scale * sum(leaves) // len(leaves)  # center of gravity
-            if minidx < maxidx and not minidx < center < maxidx:
-                center = sum(candidates) // len(candidates)
-            if max(candidates) - min(candidates) > 2 * scale:
-                center -= center % scale  # round to unscaled coordinate
-                if minidx < maxidx and not minidx < center < maxidx:
-                    center += scale
-            if ids[m] == 0:
-                startoflevel = len(matrix)
-            for rowidx in range(startoflevel, len(matrix) + 1):
-                if rowidx == len(matrix):  # need to add a new row
-                    matrix.append(
-                        [
-                            vertline if a not in (corner, None) else None
-                            for a in matrix[-1]
-                        ]
-                    )
-                row = matrix[rowidx]
-                i = j = center
-                if len(children[m]) == 1:  # place unaries directly above child
-                    return rowidx, next(iter(children[m]))[1]
-                elif all(
-                    a is None or a == vertline
-                    for a in row[min(candidates) : max(candidates) + 1]
-                ):
-                    # find free column
-                    for n in range(scale):
-                        i = j = center + n
-                        while j > minidx or i < maxidx:
-                            if i < maxidx and (
-                                matrix[rowidx][i] is None or i in candidates
-                            ):
-                                return rowidx, i
-                            elif j > minidx and (
-                                matrix[rowidx][j] is None or j in candidates
-                            ):
-                                return rowidx, j
-                            i += scale
-                            j -= scale
-            raise ValueError(
-                "could not find a free cell for:\n%s\n%s"
-                "min=%d; max=%d" % (tree[m], minidx, maxidx, dumpmatrix())
-            )
-
-        def dumpmatrix():
-            """Dump matrix contents for debugging purposes."""
-            return "\n".join(
-                "%2d: %s" % (n, " ".join(("%2r" % i)[:2] for i in row))
-                for n, row in enumerate(matrix)
-            )
-
-        leaves = tree.leaves()
-        if not all(isinstance(n, int) for n in leaves):
-            raise ValueError("All leaves must be integer indices.")
-        if len(leaves) != len(set(leaves)):
-            raise ValueError("Indices must occur at most once.")
-        if not all(0 <= n < len(sentence) for n in leaves):
-            raise ValueError(
-                "All leaves must be in the interval 0..n "
-                "with n=len(sentence)\ntokens: %d indices: "
-                "%r\nsentence: %s" % (len(sentence), tree.leaves(), sentence)
-            )
-        vertline, corner = -1, -2  # constants
-        tree = tree.copy(True)
-        for a in tree.subtrees():
-            a.sort(key=lambda n: min(n.leaves()) if isinstance(n, Tree) else n)
-        scale = 2
-        crossed = set()
-        # internal nodes and lexical nodes (no frontiers)
-        positions = tree.treepositions()
-        maxdepth = max(map(len, positions)) + 1
-        childcols = defaultdict(set)
-        matrix = [[None] * (len(sentence) * scale)]
-        nodes = {}
-        ids = {a: n for n, a in enumerate(positions)}
-        highlighted_nodes = {
-            n for a, n in ids.items() if not highlight or tree[a] in highlight
-        }
-        levels = {n: [] for n in range(maxdepth - 1)}
-        terminals = []
-        for a in positions:
-            node = tree[a]
-            if isinstance(node, Tree):
-                levels[maxdepth - node.height()].append(a)
-            else:
-                terminals.append(a)
-
-        for n in levels:
-            levels[n].sort(key=lambda n: max(tree[n].leaves()) - min(tree[n].leaves()))
-        terminals.sort()
-        positions = set(positions)
-
-        for m in terminals:
-            i = int(tree[m]) * scale
-            assert matrix[0][i] is None, (matrix[0][i], m, i)
-            matrix[0][i] = ids[m]
-            nodes[ids[m]] = sentence[tree[m]]
-            if nodes[ids[m]] is None:
-                nodes[ids[m]] = "..."
-                highlighted_nodes.discard(ids[m])
-            positions.remove(m)
-            childcols[m[:-1]].add((0, i))
-
-        # add other nodes centered on their children,
-        # if the center is already taken, back off
-        # to the left and right alternately, until an empty cell is found.
-        for n in sorted(levels, reverse=True):
-            nodesatdepth = levels[n]
-            startoflevel = len(matrix)
-            matrix.append(
-                [vertline if a not in (corner, None) else None for a in matrix[-1]]
-            )
-            for m in nodesatdepth:  # [::-1]:
-                if n < maxdepth - 1 and childcols[m]:
-                    _, pivot = min(childcols[m], key=itemgetter(1))
-                    if {
-                        a[:-1]
-                        for row in matrix[:-1]
-                        for a in row[:pivot]
-                        if isinstance(a, tuple)
-                    } & {
-                        a[:-1]
-                        for row in matrix[:-1]
-                        for a in row[pivot:]
-                        if isinstance(a, tuple)
-                    }:
-                        crossed.add(m)
-
-                rowidx, i = findcell(m, matrix, startoflevel, childcols)
-                positions.remove(m)
-
-                # block positions where children of this node branch out
-                for _, x in childcols[m]:
-                    matrix[rowidx][x] = corner
-                # assert m == () or matrix[rowidx][i] in (None, corner), (
-                #         matrix[rowidx][i], m, str(tree), ' '.join(sentence))
-                # node itself
-                matrix[rowidx][i] = ids[m]
-                nodes[ids[m]] = tree[m]
-                # add column to the set of children for its parent
-                if m != ():
-                    childcols[m[:-1]].add((rowidx, i))
-        assert len(positions) == 0
-
-        # remove unused columns, right to left
-        for m in range(scale * len(sentence) - 1, -1, -1):
-            if not any(isinstance(row[m], (Tree, int)) for row in matrix):
-                for row in matrix:
-                    del row[m]
-
-        # remove unused rows, reverse
-        matrix = [
-            row
-            for row in reversed(matrix)
-            if not all(a is None or a == vertline for a in row)
-        ]
-
-        # collect coordinates of nodes
-        coords = {}
-        for n, _ in enumerate(matrix):
-            for m, i in enumerate(matrix[n]):
-                if isinstance(i, int) and i >= 0:
-                    coords[i] = n, m
-
-        # move crossed edges last
-        positions = sorted(
-            (a for level in levels.values() for a in level),
-            key=lambda a: a[:-1] in crossed,
-        )
-
-        # collect edges from node to node
-        edges = OrderedDict()
-        for i in reversed(positions):
-            for j, _ in enumerate(tree[i]):
-                edges[ids[i + (j,)]] = ids[i]
-
-        return nodes, coords, edges, highlighted_nodes
-
-    def text(
-        self,
-        nodedist=1,
-        unicodelines=False,
-        html=False,
-        ansi=False,
-        nodecolor="blue",
-        leafcolor="red",
-        funccolor="green",
-        abbreviate=None,
-        maxwidth=16,
-    ):
-        """
-        :return: ASCII art for a discontinuous tree.
-
-        :param unicodelines: whether to use Unicode line drawing characters
-            instead of plain (7-bit) ASCII.
-        :param html: whether to wrap output in html code (default plain text).
-        :param ansi: whether to produce colors with ANSI escape sequences
-            (only effective when html==False).
-        :param leafcolor, nodecolor: specify colors of leaves and phrasal
-            nodes; effective when either html or ansi is True.
-        :param abbreviate: if True, abbreviate labels longer than 5 characters.
-            If integer, abbreviate labels longer than `abbr` characters.
-        :param maxwidth: maximum number of characters before a label starts to
-            wrap; pass None to disable.
-        """
-        if abbreviate == True:
-            abbreviate = 5
-        if unicodelines:
-            horzline = "\u2500"
-            leftcorner = "\u250c"
-            rightcorner = "\u2510"
-            vertline = " \u2502 "
-            tee = horzline + "\u252C" + horzline
-            bottom = horzline + "\u2534" + horzline
-            cross = horzline + "\u253c" + horzline
-            ellipsis = "\u2026"
-        else:
-            horzline = "_"
-            leftcorner = rightcorner = " "
-            vertline = " | "
-            tee = 3 * horzline
-            cross = bottom = "_|_"
-            ellipsis = "."
-
-        def crosscell(cur, x=vertline):
-            """Overwrite center of this cell with a vertical branch."""
-            splitl = len(cur) - len(cur) // 2 - len(x) // 2 - 1
-            lst = list(cur)
-            lst[splitl : splitl + len(x)] = list(x)
-            return "".join(lst)
-
-        result = []
-        matrix = defaultdict(dict)
-        maxnodewith = defaultdict(lambda: 3)
-        maxnodeheight = defaultdict(lambda: 1)
-        maxcol = 0
-        minchildcol = {}
-        maxchildcol = {}
-        childcols = defaultdict(set)
-        labels = {}
-        wrapre = re.compile(
-            "(.{%d,%d}\\b\\W*|.{%d})" % (maxwidth - 4, maxwidth, maxwidth)
-        )
-        # collect labels and coordinates
-        for a in self.nodes:
-            row, column = self.coords[a]
-            matrix[row][column] = a
-            maxcol = max(maxcol, column)
-            label = (
-                self.nodes[a].label()
-                if isinstance(self.nodes[a], Tree)
-                else self.nodes[a]
-            )
-            if abbreviate and len(label) > abbreviate:
-                label = label[:abbreviate] + ellipsis
-            if maxwidth and len(label) > maxwidth:
-                label = wrapre.sub(r"\1\n", label).strip()
-            label = label.split("\n")
-            maxnodeheight[row] = max(maxnodeheight[row], len(label))
-            maxnodewith[column] = max(maxnodewith[column], max(map(len, label)))
-            labels[a] = label
-            if a not in self.edges:
-                continue  # e.g., root
-            parent = self.edges[a]
-            childcols[parent].add((row, column))
-            minchildcol[parent] = min(minchildcol.get(parent, column), column)
-            maxchildcol[parent] = max(maxchildcol.get(parent, column), column)
-        # bottom up level order traversal
-        for row in sorted(matrix, reverse=True):
-            noderows = [
-                ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
-                for _ in range(maxnodeheight[row])
-            ]
-            branchrow = ["".center(maxnodewith[col]) for col in range(maxcol + 1)]
-            for col in matrix[row]:
-                n = matrix[row][col]
-                node = self.nodes[n]
-                text = labels[n]
-                if isinstance(node, Tree):
-                    # draw horizontal branch towards children for this node
-                    if n in minchildcol and minchildcol[n] < maxchildcol[n]:
-                        i, j = minchildcol[n], maxchildcol[n]
-                        a, b = (maxnodewith[i] + 1) // 2 - 1, maxnodewith[j] // 2
-                        branchrow[i] = ((" " * a) + leftcorner).ljust(
-                            maxnodewith[i], horzline
-                        )
-                        branchrow[j] = (rightcorner + (" " * b)).rjust(
-                            maxnodewith[j], horzline
-                        )
-                        for i in range(minchildcol[n] + 1, maxchildcol[n]):
-                            if i == col and any(a == i for _, a in childcols[n]):
-                                line = cross
-                            elif i == col:
-                                line = bottom
-                            elif any(a == i for _, a in childcols[n]):
-                                line = tee
-                            else:
-                                line = horzline
-                            branchrow[i] = line.center(maxnodewith[i], horzline)
-                    else:  # if n and n in minchildcol:
-                        branchrow[col] = crosscell(branchrow[col])
-                text = [a.center(maxnodewith[col]) for a in text]
-                color = nodecolor if isinstance(node, Tree) else leafcolor
-                if isinstance(node, Tree) and node.label().startswith("-"):
-                    color = funccolor
-                if html:
-                    text = [escape(a, quote=False) for a in text]
-                    if n in self.highlight:
-                        text = [f"<font color={color}>{a}</font>" for a in text]
-                elif ansi and n in self.highlight:
-                    text = ["\x1b[%d;1m%s\x1b[0m" % (ANSICOLOR[color], a) for a in text]
-                for x in range(maxnodeheight[row]):
-                    # draw vertical lines in partially filled multiline node
-                    # labels, but only if it's not a frontier node.
-                    noderows[x][col] = (
-                        text[x]
-                        if x < len(text)
-                        else (vertline if childcols[n] else " ").center(
-                            maxnodewith[col], " "
-                        )
-                    )
-            # for each column, if there is a node below us which has a parent
-            # above us, draw a vertical branch in that column.
-            if row != max(matrix):
-                for n, (childrow, col) in self.coords.items():
-                    if n > 0 and self.coords[self.edges[n]][0] < row < childrow:
-                        branchrow[col] = crosscell(branchrow[col])
-                        if col not in matrix[row]:
-                            for noderow in noderows:
-                                noderow[col] = crosscell(noderow[col])
-                branchrow = [
-                    a + ((a[-1] if a[-1] != " " else b[0]) * nodedist)
-                    for a, b in zip(branchrow, branchrow[1:] + [" "])
-                ]
-                result.append("".join(branchrow))
-            result.extend(
-                (" " * nodedist).join(noderow) for noderow in reversed(noderows)
-            )
-        return "\n".join(reversed(result)) + "\n"
-
-    def svg(self, nodecolor="blue", leafcolor="red", funccolor="green"):
-        """
-        :return: SVG representation of a tree.
-        """
-        fontsize = 12
-        hscale = 40
-        vscale = 25
-        hstart = vstart = 20
-        width = max(col for _, col in self.coords.values())
-        height = max(row for row, _ in self.coords.values())
-        result = [
-            '<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
-            'width="%dem" height="%dem" viewBox="%d %d %d %d">'
-            % (
-                width * 3,
-                height * 2.5,
-                -hstart,
-                -vstart,
-                width * hscale + 3 * hstart,
-                height * vscale + 3 * vstart,
-            )
-        ]
-
-        children = defaultdict(set)
-        for n in self.nodes:
-            if n:
-                children[self.edges[n]].add(n)
-
-        # horizontal branches from nodes to children
-        for node in self.nodes:
-            if not children[node]:
-                continue
-            y, x = self.coords[node]
-            x *= hscale
-            y *= vscale
-            x += hstart
-            y += vstart + fontsize // 2
-            childx = [self.coords[c][1] for c in children[node]]
-            xmin = hstart + hscale * min(childx)
-            xmax = hstart + hscale * max(childx)
-            result.append(
-                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
-                'points="%g,%g %g,%g" />' % (xmin, y, xmax, y)
-            )
-            result.append(
-                '\t<polyline style="stroke:black; stroke-width:1; fill:none;" '
-                'points="%g,%g %g,%g" />' % (x, y, x, y - fontsize // 3)
-            )
-
-        # vertical branches from children to parents
-        for child, parent in self.edges.items():
-            y, _ = self.coords[parent]
-            y *= vscale
-            y += vstart + fontsize // 2
-            childy, childx = self.coords[child]
-            childx *= hscale
-            childy *= vscale
-            childx += hstart
-            childy += vstart - fontsize
-            result += [
-                '\t<polyline style="stroke:white; stroke-width:10; fill:none;"'
-                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y + 5),
-                '\t<polyline style="stroke:black; stroke-width:1; fill:none;"'
-                ' points="%g,%g %g,%g" />' % (childx, childy, childx, y),
-            ]
-
-        # write nodes with coordinates
-        for n, (row, column) in self.coords.items():
-            node = self.nodes[n]
-            x = column * hscale + hstart
-            y = row * vscale + vstart
-            if n in self.highlight:
-                color = nodecolor if isinstance(node, Tree) else leafcolor
-                if isinstance(node, Tree) and node.label().startswith("-"):
-                    color = funccolor
-            else:
-                color = "black"
-            result += [
-                '\t<text style="text-anchor: middle; fill: %s; '
-                'font-size: %dpx;" x="%g" y="%g">%s</text>'
-                % (
-                    color,
-                    fontsize,
-                    x,
-                    y,
-                    escape(
-                        node.label() if isinstance(node, Tree) else node, quote=False
-                    ),
-                )
-            ]
-
-        result += ["</svg>"]
-        return "\n".join(result)
-
-
-def test():
-    """Do some tree drawing tests."""
-
-    def print_tree(n, tree, sentence=None, ansi=True, **xargs):
-        print()
-        print('{}: "{}"'.format(n, " ".join(sentence or tree.leaves())))
-        print(tree)
-        print()
-        drawtree = TreePrettyPrinter(tree, sentence)
-        try:
-            print(drawtree.text(unicodelines=ansi, ansi=ansi, **xargs))
-        except (UnicodeDecodeError, UnicodeEncodeError):
-            print(drawtree.text(unicodelines=False, ansi=False, **xargs))
-
-    from nltk.corpus import treebank
-
-    for n in [0, 1440, 1591, 2771, 2170]:
-        tree = treebank.parsed_sents()[n]
-        print_tree(n, tree, nodedist=2, maxwidth=8)
-    print()
-    print("ASCII version:")
-    print(TreePrettyPrinter(tree).text(nodedist=2))
-
-    tree = Tree.fromstring(
-        "(top (punct 8) (smain (noun 0) (verb 1) (inf (verb 5) (inf (verb 6) "
-        "(conj (inf (pp (prep 2) (np (det 3) (noun 4))) (verb 7)) (inf (verb 9)) "
-        "(vg 10) (inf (verb 11)))))) (punct 12))",
-        read_leaf=int,
-    )
-    sentence = (
-        "Ze had met haar moeder kunnen gaan winkelen ,"
-        " zwemmen of terrassen .".split()
-    )
-    print_tree("Discontinuous tree", tree, sentence, nodedist=2)
+class TreePrettyPrinter(Deprecated, TPP):
+    """Import `TreePrettyPrinter` using `from nltk.tree import TreePrettyPrinter` instead."""
 
 
 __all__ = ["TreePrettyPrinter"]
-
-if __name__ == "__main__":
-    test()
diff --git a/nltk/treetransforms.py b/nltk/treetransforms.py
index 5b89d1a01..d1360618d 100644
--- a/nltk/treetransforms.py
+++ b/nltk/treetransforms.py
@@ -107,232 +107,20 @@ The following is a short tutorial on the available transformations.
 
 """
 
-from nltk.tree import Tree
+from nltk.internals import deprecated
+from nltk.tree.transforms import chomsky_normal_form as cnf
+from nltk.tree.transforms import collapse_unary as cu
+from nltk.tree.transforms import un_chomsky_normal_form as ucnf
+
+chomsky_normal_form = deprecated(
+    "Import using `from nltk.tree import chomsky_normal_form` instead."
+)(cnf)
+un_chomsky_normal_form = deprecated(
+    "Import using `from nltk.tree import un_chomsky_normal_form` instead."
+)(ucnf)
+collapse_unary = deprecated(
+    "Import using `from nltk.tree import collapse_unary` instead."
+)(cu)
 
 
-def chomsky_normal_form(
-    tree, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^"
-):
-    # assume all subtrees have homogeneous children
-    # assume all terminals have no siblings
-
-    # A semi-hack to have elegant looking code below.  As a result,
-    # any subtree with a branching factor greater than 999 will be incorrectly truncated.
-    if horzMarkov is None:
-        horzMarkov = 999
-
-    # Traverse the tree depth-first keeping a list of ancestor nodes to the root.
-    # I chose not to use the tree.treepositions() method since it requires
-    # two traversals of the tree (one to get the positions, one to iterate
-    # over them) and node access time is proportional to the height of the node.
-    # This method is 7x faster which helps when parsing 40,000 sentences.
-
-    nodeList = [(tree, [tree.label()])]
-    while nodeList != []:
-        node, parent = nodeList.pop()
-        if isinstance(node, Tree):
-
-            # parent annotation
-            parentString = ""
-            originalNode = node.label()
-            if vertMarkov != 0 and node != tree and isinstance(node[0], Tree):
-                parentString = "{}<{}>".format(parentChar, "-".join(parent))
-                node.set_label(node.label() + parentString)
-                parent = [originalNode] + parent[: vertMarkov - 1]
-
-            # add children to the agenda before we mess with them
-            for child in node:
-                nodeList.append((child, parent))
-
-            # chomsky normal form factorization
-            if len(node) > 2:
-                childNodes = [child.label() for child in node]
-                nodeCopy = node.copy()
-                node[0:] = []  # delete the children
-
-                curNode = node
-                numChildren = len(nodeCopy)
-                for i in range(1, numChildren - 1):
-                    if factor == "right":
-                        newHead = "{}{}<{}>{}".format(
-                            originalNode,
-                            childChar,
-                            "-".join(
-                                childNodes[i : min([i + horzMarkov, numChildren])]
-                            ),
-                            parentString,
-                        )  # create new head
-                        newNode = Tree(newHead, [])
-                        curNode[0:] = [nodeCopy.pop(0), newNode]
-                    else:
-                        newHead = "{}{}<{}>{}".format(
-                            originalNode,
-                            childChar,
-                            "-".join(
-                                childNodes[max([numChildren - i - horzMarkov, 0]) : -i]
-                            ),
-                            parentString,
-                        )
-                        newNode = Tree(newHead, [])
-                        curNode[0:] = [newNode, nodeCopy.pop()]
-
-                    curNode = newNode
-
-                curNode[0:] = [child for child in nodeCopy]
-
-
-def un_chomsky_normal_form(
-    tree, expandUnary=True, childChar="|", parentChar="^", unaryChar="+"
-):
-    # Traverse the tree-depth first keeping a pointer to the parent for modification purposes.
-    nodeList = [(tree, [])]
-    while nodeList != []:
-        node, parent = nodeList.pop()
-        if isinstance(node, Tree):
-            # if the node contains the 'childChar' character it means that
-            # it is an artificial node and can be removed, although we still need
-            # to move its children to its parent
-            childIndex = node.label().find(childChar)
-            if childIndex != -1:
-                nodeIndex = parent.index(node)
-                parent.remove(parent[nodeIndex])
-                # Generated node was on the left if the nodeIndex is 0 which
-                # means the grammar was left factored.  We must insert the children
-                # at the beginning of the parent's children
-                if nodeIndex == 0:
-                    parent.insert(0, node[0])
-                    parent.insert(1, node[1])
-                else:
-                    parent.extend([node[0], node[1]])
-
-                # parent is now the current node so the children of parent will be added to the agenda
-                node = parent
-            else:
-                parentIndex = node.label().find(parentChar)
-                if parentIndex != -1:
-                    # strip the node name of the parent annotation
-                    node.set_label(node.label()[:parentIndex])
-
-                # expand collapsed unary productions
-                if expandUnary == True:
-                    unaryIndex = node.label().find(unaryChar)
-                    if unaryIndex != -1:
-                        newNode = Tree(
-                            node.label()[unaryIndex + 1 :], [i for i in node]
-                        )
-                        node.set_label(node.label()[:unaryIndex])
-                        node[0:] = [newNode]
-
-            for child in node:
-                nodeList.append((child, node))
-
-
-def collapse_unary(tree, collapsePOS=False, collapseRoot=False, joinChar="+"):
-    """
-    Collapse subtrees with a single child (ie. unary productions)
-    into a new non-terminal (Tree node) joined by 'joinChar'.
-    This is useful when working with algorithms that do not allow
-    unary productions, and completely removing the unary productions
-    would require loss of useful information.  The Tree is modified
-    directly (since it is passed by reference) and no value is returned.
-
-    :param tree: The Tree to be collapsed
-    :type  tree: Tree
-    :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie.
-                        Part-of-Speech tags) since they are always unary productions
-    :type  collapsePOS: bool
-    :param collapseRoot: 'False' (default) will not modify the root production
-                         if it is unary.  For the Penn WSJ treebank corpus, this corresponds
-                         to the TOP -> productions.
-    :type collapseRoot: bool
-    :param joinChar: A string used to connect collapsed node values (default = "+")
-    :type  joinChar: str
-    """
-
-    if collapseRoot == False and isinstance(tree, Tree) and len(tree) == 1:
-        nodeList = [tree[0]]
-    else:
-        nodeList = [tree]
-
-    # depth-first traversal of tree
-    while nodeList != []:
-        node = nodeList.pop()
-        if isinstance(node, Tree):
-            if (
-                len(node) == 1
-                and isinstance(node[0], Tree)
-                and (collapsePOS == True or isinstance(node[0, 0], Tree))
-            ):
-                node.set_label(node.label() + joinChar + node[0].label())
-                node[0:] = [child for child in node[0]]
-                # since we assigned the child's children to the current node,
-                # evaluate the current node again
-                nodeList.append(node)
-            else:
-                for child in node:
-                    nodeList.append(child)
-
-
-#################################################################
-# Demonstration
-#################################################################
-
-
-def demo():
-    """
-    A demonstration showing how each tree transform can be used.
-    """
-
-    from copy import deepcopy
-
-    from nltk import tree, treetransforms
-    from nltk.draw.tree import draw_trees
-
-    # original tree from WSJ bracketed text
-    sentence = """(TOP
-  (S
-    (S
-      (VP
-        (VBN Turned)
-        (ADVP (RB loose))
-        (PP
-          (IN in)
-          (NP
-            (NP (NNP Shane) (NNP Longman) (POS 's))
-            (NN trading)
-            (NN room)))))
-    (, ,)
-    (NP (DT the) (NN yuppie) (NNS dealers))
-    (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
-    (. .)))"""
-    t = tree.Tree.fromstring(sentence, remove_empty_top_bracketing=True)
-
-    # collapse subtrees with only one child
-    collapsedTree = deepcopy(t)
-    treetransforms.collapse_unary(collapsedTree)
-
-    # convert the tree to CNF
-    cnfTree = deepcopy(collapsedTree)
-    treetransforms.chomsky_normal_form(cnfTree)
-
-    # convert the tree to CNF with parent annotation (one level) and horizontal smoothing of order two
-    parentTree = deepcopy(collapsedTree)
-    treetransforms.chomsky_normal_form(parentTree, horzMarkov=2, vertMarkov=1)
-
-    # convert the tree back to its original form (used to make CYK results comparable)
-    original = deepcopy(parentTree)
-    treetransforms.un_chomsky_normal_form(original)
-
-    # convert tree back to bracketed text
-    sentence2 = original.pprint()
-    print(sentence)
-    print(sentence2)
-    print("Sentences the same? ", sentence == sentence2)
-
-    draw_trees(t, collapsedTree, cnfTree, parentTree, original)
-
-
-if __name__ == "__main__":
-    demo()
-
 __all__ = ["chomsky_normal_form", "un_chomsky_normal_form", "collapse_unary"]


Test patch:
diff --git a/nltk/test/bleu.doctest b/nltk/test/bleu.doctest
index a116dcb61..4ea6b8ac9 100644
--- a/nltk/test/bleu.doctest
+++ b/nltk/test/bleu.doctest
@@ -9,7 +9,7 @@ If the candidate has no alignment to any of the references, the BLEU score is 0.
 >>> bleu(
 ...     ['The candidate has no alignment to any of the references'.split()],
 ...     'John loves Mary'.split(),
-...     [1],
+...     (1,),
 ... )
 0
 
diff --git a/nltk/test/generate.doctest b/nltk/test/generate.doctest
index a23f53a9c..ff9b96ddb 100644
--- a/nltk/test/generate.doctest
+++ b/nltk/test/generate.doctest
@@ -64,3 +64,15 @@ The number of sentences of different max depths:
     114
     >>> len(list(generate(grammar)))
     114
+
+Infinite grammars will throw a RecursionError when not bounded by some ``depth``:
+
+    >>> grammar = CFG.fromstring("""
+    ... S -> A B
+    ... A -> B
+    ... B -> "b" | A
+    ... """)
+    >>> list(generate(grammar))
+    Traceback (most recent call last):
+    ...
+    RuntimeError: The grammar has rule(s) that yield infinite recursion!
diff --git a/nltk/test/grammar.doctest b/nltk/test/grammar.doctest
index 91349bc81..fce33264a 100644
--- a/nltk/test/grammar.doctest
+++ b/nltk/test/grammar.doctest
@@ -46,3 +46,24 @@ Chomsky Normal Form grammar (Test for bug 474)
     >>> g = CFG.fromstring("VP^<TOP> -> VBP NP^<VP-TOP>")
     >>> g.productions()[0].lhs()
     VP^<TOP>
+
+Grammars can contain both empty strings and empty productions:
+
+    >>> from nltk.grammar import CFG
+    >>> from nltk.parse.generate import generate
+    >>> grammar = CFG.fromstring("""
+    ... S -> A B
+    ... A -> 'a'
+    ... # An empty string:
+    ... B -> 'b' | ''
+    ... """)
+    >>> list(generate(grammar))
+    [['a', 'b'], ['a', '']]
+    >>> grammar = CFG.fromstring("""
+    ... S -> A B
+    ... A -> 'a'
+    ... # An empty production:
+    ... B -> 'b' |
+    ... """)
+    >>> list(generate(grammar))
+    [['a', 'b'], ['a']]
diff --git a/nltk/test/parse.doctest b/nltk/test/parse.doctest
index 818783bde..187a1ca14 100644
--- a/nltk/test/parse.doctest
+++ b/nltk/test/parse.doctest
@@ -545,7 +545,43 @@ Unit tests for the Probabilistic CFG class
 
     >>> from nltk.corpus import treebank
     >>> from itertools import islice
-    >>> from nltk.grammar import PCFG, induce_pcfg, toy_pcfg1, toy_pcfg2
+    >>> from nltk.grammar import PCFG, induce_pcfg
+    >>> toy_pcfg1 = PCFG.fromstring("""
+    ...     S -> NP VP [1.0]
+    ...     NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
+    ...     Det -> 'the' [0.8] | 'my' [0.2]
+    ...     N -> 'man' [0.5] | 'telescope' [0.5]
+    ...     VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
+    ...     V -> 'ate' [0.35] | 'saw' [0.65]
+    ...     PP -> P NP [1.0]
+    ...     P -> 'with' [0.61] | 'under' [0.39]
+    ...     """)
+
+    >>> toy_pcfg2 = PCFG.fromstring("""
+    ...     S    -> NP VP         [1.0]
+    ...     VP   -> V NP          [.59]
+    ...     VP   -> V             [.40]
+    ...     VP   -> VP PP         [.01]
+    ...     NP   -> Det N         [.41]
+    ...     NP   -> Name          [.28]
+    ...     NP   -> NP PP         [.31]
+    ...     PP   -> P NP          [1.0]
+    ...     V    -> 'saw'         [.21]
+    ...     V    -> 'ate'         [.51]
+    ...     V    -> 'ran'         [.28]
+    ...     N    -> 'boy'         [.11]
+    ...     N    -> 'cookie'      [.12]
+    ...     N    -> 'table'       [.13]
+    ...     N    -> 'telescope'   [.14]
+    ...     N    -> 'hill'        [.5]
+    ...     Name -> 'Jack'        [.52]
+    ...     Name -> 'Bob'         [.48]
+    ...     P    -> 'with'        [.61]
+    ...     P    -> 'under'       [.39]
+    ...     Det  -> 'the'         [.41]
+    ...     Det  -> 'a'           [.31]
+    ...     Det  -> 'my'          [.28]
+    ...     """)
 
 Create a set of PCFG productions.
 
diff --git a/nltk/test/tokenize.doctest b/nltk/test/tokenize.doctest
index 0e3a0749e..ec980711b 100644
--- a/nltk/test/tokenize.doctest
+++ b/nltk/test/tokenize.doctest
@@ -4,7 +4,7 @@
     >>> from nltk.tokenize import *
 
 Regression Tests: NLTKWordTokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Tokenizing some test strings.
 
@@ -259,6 +259,19 @@ It should not hang on long sequences of the same punctuation character.
     >>> tknzr.tokenize(s10)
     ['Photo', ':', "Aujourd'hui", 'sur', 'http://t.co/0gebOFDUzn', 'Projet', '...', 'http://t.co/bKfIUbydz2', '...', 'http://fb.me/3b6uXpz0L']
 
+Tokenizing multiple sentences at once:
+
+    >>> tknzr = TweetTokenizer()
+    >>> sentences = [
+    ...     "This is a cooool #dummysmiley: :-) :-P <3 and some arrows < > -> <--",
+    ...     "@jrmy: I'm REALLY HAPPYYY about that! NICEEEE :D :P",
+    ...     "@_willy65: No place for @chuck tonight. Sorry."
+    ... ]
+    >>> tknzr.tokenize_sents(sentences) # doctest: +NORMALIZE_WHITESPACE
+    [['This', 'is', 'a', 'cooool', '#dummysmiley', ':', ':-)', ':-P', '<3', 'and', 'some', 'arrows', '<', '>', '->', '<--'],
+    ['@jrmy', ':', "I'm", 'REALLY', 'HAPPYYY', 'about', 'that', '!', 'NICEEEE', ':D', ':P'],
+    ['@_willy65', ':', 'No', 'place', 'for', '@chuck', 'tonight', '.', 'Sorry', '.']]
+
 
 Regression Tests: PunktSentenceTokenizer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/nltk/test/tree.doctest b/nltk/test/tree.doctest
index a47927328..feea8201b 100644
--- a/nltk/test/tree.doctest
+++ b/nltk/test/tree.doctest
@@ -113,7 +113,7 @@ type:
     >>> print(tree)
     (VP (V enjoyed) (NP my cookie))
     >>> print(type(tree))
-    <class 'nltk.tree.ImmutableTree'>
+    <class 'nltk.tree.immutable.ImmutableTree'>
     >>> tree[1] = 'x'
     Traceback (most recent call last):
       . . .
@@ -345,7 +345,7 @@ Parented trees can be created from strings using the classmethod
     >>> print(ptree)
     (VP (VERB saw) (NP (DET the) (NOUN dog)))
     >>> print(type(ptree))
-    <class 'nltk.tree.ParentedTree'>
+    <class 'nltk.tree.parented.ParentedTree'>
 
 Parented trees can also be created by using the classmethod
 `ParentedTree.convert` to convert another type of tree to a parented
@@ -356,7 +356,7 @@ tree:
     >>> print(ptree)
     (VP (VERB saw) (NP (DET the) (NOUN dog)))
     >>> print(type(ptree))
-    <class 'nltk.tree.ParentedTree'>
+    <class 'nltk.tree.parented.ParentedTree'>
 
 .. clean-up:
 
@@ -802,13 +802,59 @@ Test that a tree can not be given multiple parents:
 
 [more to be written]
 
+Shallow copying can be tricky for Tree and several of its subclasses.
+For shallow copies of Tree, only the root node is reconstructed, while
+all the children are shared between the two trees. Modify the children
+of one tree - and the shallowly copied tree will also update.
+
+    >>> from nltk.tree import Tree, ParentedTree, MultiParentedTree
+    >>> tree = Tree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> copy_tree = tree.copy(deep=False)
+    >>> tree == copy_tree # Ensure identical labels and nodes
+    True
+    >>> id(copy_tree[0]) == id(tree[0]) # Ensure shallow copy - the children are the same objects in memory
+    True
+
+For ParentedTree objects, this behaviour is not possible. With a shallow
+copy, the children of the root node would be reused for both the original
+and the shallow copy. For this to be possible, some children would need
+to have multiple parents. As this is forbidden for ParentedTree objects,
+attempting to make a shallow copy will cause a warning, and a deep copy
+is made instead.
+
+    >>> ptree = ParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> copy_ptree = ptree.copy(deep=False)
+    >>> copy_ptree == ptree # Ensure identical labels and nodes
+    True
+    >>> id(copy_ptree[0]) != id(ptree[0]) # Shallow copying isn't supported - it defaults to deep copy.
+    True
+
+For MultiParentedTree objects, the issue of only allowing one parent that
+can be seen for ParentedTree objects is no more. Shallow copying a
+MultiParentedTree gives the children of the root node two parents:
+the original and the newly copied root.
+
+    >>> mptree = MultiParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> len(mptree[0].parents())
+    1
+    >>> copy_mptree = mptree.copy(deep=False)
+    >>> copy_mptree == mptree # Ensure identical labels and nodes
+    True
+    >>> len(mptree[0].parents())
+    2
+    >>> len(copy_mptree[0].parents())
+    2
+
+Shallow copying a MultiParentedTree is similar to creating a second root
+which is identically labeled as the root on which the copy method was called.
+
 
 ImmutableParentedTree Regression Tests
 --------------------------------------
 
     >>> iptree = ImmutableParentedTree.convert(ptree)
     >>> type(iptree)
-    <class 'nltk.tree.ImmutableParentedTree'>
+    <class 'nltk.tree.immutable.ImmutableParentedTree'>
     >>> del iptree[0]
     Traceback (most recent call last):
       . . .
@@ -1110,7 +1156,7 @@ ImmutableMultiParentedTree Regression Tests
 
     >>> imptree = ImmutableMultiParentedTree.convert(mptree)
     >>> type(imptree)
-    <class 'nltk.tree.ImmutableMultiParentedTree'>
+    <class 'nltk.tree.immutable.ImmutableMultiParentedTree'>
     >>> del imptree[0]
     Traceback (most recent call last):
       . . .
@@ -1137,7 +1183,7 @@ ProbabilisticTree Regression Tests
 
     >>> imprtree = ImmutableProbabilisticTree.convert(prtree)
     >>> type(imprtree)
-    <class 'nltk.tree.ImmutableProbabilisticTree'>
+    <class 'nltk.tree.immutable.ImmutableProbabilisticTree'>
     >>> del imprtree[0]
     Traceback (most recent call last):
       . . .
@@ -1155,3 +1201,23 @@ This used to discard the ``(B b)`` subtree (fixed in svn 6270):
 
     >>> print(Tree.fromstring('((A a) (B b))'))
     ( (A a) (B b))
+
+Pickling ParentedTree instances didn't work for Python 3.7 onwards (See #2478)
+
+    >>> import pickle
+    >>> tree = ParentedTree.fromstring('(S (NN x) (NP x) (NN x))')
+    >>> print(tree)
+    (S (NN x) (NP x) (NN x))
+
+    >>> pickled = pickle.dumps(tree)
+    >>> tree_loaded = pickle.loads(pickled)
+    >>> print(tree_loaded)
+    (S (NN x) (NP x) (NN x))
+
+ParentedTree used to be impossible to (deep)copy. (See #1324)
+
+    >>> from nltk.tree import ParentedTree
+    >>> import copy
+    >>> tree = ParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))")
+    >>> tree == copy.deepcopy(tree) == copy.copy(tree) == tree.copy(deep=True) == tree.copy()
+    True
diff --git a/nltk/test/treeprettyprinter.doctest b/nltk/test/treeprettyprinter.doctest
index cea049c98..9f11af21e 100644
--- a/nltk/test/treeprettyprinter.doctest
+++ b/nltk/test/treeprettyprinter.doctest
@@ -1,12 +1,11 @@
 .. Copyright (C) 2001-2021 NLTK Project
 .. For license information, see LICENSE.TXT
 
-========================================================
- Unit tests for nltk.treeprettyprinter.TreePrettyPrinter
-========================================================
+=========================================================
+ Unit tests for nltk.tree.prettyprinter.TreePrettyPrinter
+=========================================================
 
-    >>> from nltk.tree import Tree
-    >>> from nltk.treeprettyprinter import TreePrettyPrinter
+    >>> from nltk.tree import Tree, TreePrettyPrinter
 
 Tree nr 2170 from nltk.corpus.treebank:
 
@@ -124,3 +123,55 @@ A discontinuous tree:
     noun verb prep det       noun   verb  verb       verb   punct   verb   vg    verb   punct
      │    │    │    │         │      │     │          │       │      │     │      │       │
      Ze  had  met  haar     moeder kunnen gaan     winkelen   ,   zwemmen  of terrassen   .
+
+Importing TreePrettyPrinter
+---------------------------
+
+First of all, a simple tree will be constructed::
+
+  >>> from nltk.tree import Tree
+  >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))')
+
+We'll use this sample tree to show that the method of importing `TreePrettyPrinter` work correctly:
+
+- Recommended::
+
+    >>> from nltk.tree import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+- Alternative but valid options::
+
+    >>> from nltk import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+    >>> from nltk.tree.prettyprinter import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+- Deprecated, do not use::
+
+    >>> from nltk.treeprettyprinter import TreePrettyPrinter
+    >>> print(TreePrettyPrinter(tree).text())
+          S
+      ____|____
+     NP        VP
+     |         |
+    Mary     walks
+
+  This method will throw a DeprecationWarning::
+
+    Import `TreePrettyPrinter` using `from nltk.tree import TreePrettyPrinter` instead.
diff --git a/nltk/test/treetransforms.doctest b/nltk/test/treetransforms.doctest
index a3870138d..411499340 100644
--- a/nltk/test/treetransforms.doctest
+++ b/nltk/test/treetransforms.doctest
@@ -6,8 +6,7 @@ Unit tests for the TreeTransformation class
 -------------------------------------------
 
     >>> from copy import deepcopy
-    >>> from nltk.tree import *
-    >>> from nltk.treetransforms import *
+    >>> from nltk.tree import Tree, collapse_unary, chomsky_normal_form, un_chomsky_normal_form
 
     >>> tree_string = "(TOP (S (S (VP (VBN Turned) (ADVP (RB loose)) (PP (IN in) (NP (NP (NNP Shane) (NNP Longman) (POS 's)) (NN trading) (NN room))))) (, ,) (NP (DT the) (NN yuppie) (NNS dealers)) (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right)))) (. .)))"
 
diff --git a/nltk/test/unit/translate/test_bleu.py b/nltk/test/unit/translate/test_bleu.py
index e6afbd7dc..ac3565e74 100644
--- a/nltk/test/unit/translate/test_bleu.py
+++ b/nltk/test/unit/translate/test_bleu.py
@@ -120,7 +120,7 @@ class TestBLEU(unittest.TestCase):
 
         # Test BLEU to nth order of n-grams, where n is len(hypothesis).
         for n in range(1, len(hypothesis)):
-            weights = [1.0 / n] * n  # Uniform weights.
+            weights = (1.0 / n,) * n  # Uniform weights.
             assert sentence_bleu(references, hypothesis, weights) == 0
 
     def test_full_matches(self):
@@ -130,7 +130,7 @@ class TestBLEU(unittest.TestCase):
 
         # Test BLEU to nth order of n-grams, where n is len(hypothesis).
         for n in range(1, len(hypothesis)):
-            weights = [1.0 / n] * n  # Uniform weights.
+            weights = (1.0 / n,) * n  # Uniform weights.
             assert sentence_bleu(references, hypothesis, weights) == 1.0
 
     def test_partial_matches_hypothesis_longer_than_reference(self):
@@ -153,7 +153,7 @@ class TestBLEUFringeCases(unittest.TestCase):
         references = ["John loves Mary ?".split()]
         hypothesis = "John loves Mary".split()
         n = len(hypothesis) + 1  #
-        weights = [1.0 / n] * n  # Uniform weights.
+        weights = (1.0 / n,) * n  # Uniform weights.
         # Since no n-grams matches were found the result should be zero
         # exp(w_1 * 1 * w_2 * 1 * w_3 * 1 * w_4 * -inf) = 0
         self.assertAlmostEqual(
@@ -279,3 +279,127 @@ class TestBLEUWithBadSentence(unittest.TestCase):
                 )
         except AttributeError:  # unittest.TestCase.assertWarns is only supported in Python >= 3.2.
             self.assertAlmostEqual(corpus_bleu(references, hypotheses), 0.0, places=4)
+
+
+class TestBLEUWithMultipleWeights(unittest.TestCase):
+    def test_corpus_bleu_with_multiple_weights(self):
+        hyp1 = [
+            "It",
+            "is",
+            "a",
+            "guide",
+            "to",
+            "action",
+            "which",
+            "ensures",
+            "that",
+            "the",
+            "military",
+            "always",
+            "obeys",
+            "the",
+            "commands",
+            "of",
+            "the",
+            "party",
+        ]
+        ref1a = [
+            "It",
+            "is",
+            "a",
+            "guide",
+            "to",
+            "action",
+            "that",
+            "ensures",
+            "that",
+            "the",
+            "military",
+            "will",
+            "forever",
+            "heed",
+            "Party",
+            "commands",
+        ]
+        ref1b = [
+            "It",
+            "is",
+            "the",
+            "guiding",
+            "principle",
+            "which",
+            "guarantees",
+            "the",
+            "military",
+            "forces",
+            "always",
+            "being",
+            "under",
+            "the",
+            "command",
+            "of",
+            "the",
+            "Party",
+        ]
+        ref1c = [
+            "It",
+            "is",
+            "the",
+            "practical",
+            "guide",
+            "for",
+            "the",
+            "army",
+            "always",
+            "to",
+            "heed",
+            "the",
+            "directions",
+            "of",
+            "the",
+            "party",
+        ]
+        hyp2 = [
+            "he",
+            "read",
+            "the",
+            "book",
+            "because",
+            "he",
+            "was",
+            "interested",
+            "in",
+            "world",
+            "history",
+        ]
+        ref2a = [
+            "he",
+            "was",
+            "interested",
+            "in",
+            "world",
+            "history",
+            "because",
+            "he",
+            "read",
+            "the",
+            "book",
+        ]
+        weight_1 = (1, 0, 0, 0)
+        weight_2 = (0.25, 0.25, 0.25, 0.25)
+        weight_3 = (0, 0, 0, 0, 1)
+
+        bleu_scores = corpus_bleu(
+            list_of_references=[[ref1a, ref1b, ref1c], [ref2a]],
+            hypotheses=[hyp1, hyp2],
+            weights=[weight_1, weight_2, weight_3],
+        )
+        assert bleu_scores[0] == corpus_bleu(
+            [[ref1a, ref1b, ref1c], [ref2a]], [hyp1, hyp2], weight_1
+        )
+        assert bleu_scores[1] == corpus_bleu(
+            [[ref1a, ref1b, ref1c], [ref2a]], [hyp1, hyp2], weight_2
+        )
+        assert bleu_scores[2] == corpus_bleu(
+            [[ref1a, ref1b, ref1c], [ref2a]], [hyp1, hyp2], weight_3
+        )