parser

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint.bin filter=lfs diff=lfs merge=lfs -text

app.py

@@ -3,6 +3,19 @@ import tabulate
 import matplotlib.pyplot as plt
 import networkx as nx
 
+from model import Parser
+
+
+parser = Parser()
+
+def parse(text):
+    output = parser.parse(text)
+
+    dependency_tree = render_dependency_tree(output["forms"], output["heads"], output["deprels"])
+    table = render_table(output["forms"], output["lemmas"], output["upos"], output["xpos"], output["ne"])
+
+    return dependency_tree, table
+
 
 def render_dependency_tree(words, parents, labels):
     fig, ax = plt.subplots(figsize=(32, 16))
@@ -111,12 +124,17 @@ edge_labels = [
     if line and not line.startswith("#")
 ]
 
-def render_table(forms, lemmas, upos, xpos, feats, metadata, edges, edge_labels):
+def render_table(forms, lemmas, upos, xpos, feats, named_entities):
     feats = [[f"*{f.split('=')[0]}:* {f.split('=')[1]}" for f in (feat.split("|")) if '=' in f] for feat in feats]
     max_len = max(1, max([len(feat) for feat in feats]))
     feats = [feat + [""] * (max_len - len(feat)) for feat in feats]
     feats = list(zip(*feats))
 
+    named_entities = [
+        "" if ne == "O" else f"<< {ne.split('-')[1]} >>" if ne.startswith("B") else ne.split('-')[1] if ne.startswith("I") and i - 1 < len(named_entities) and named_entities[i + 1].startswith("I") else f"{ne.split('-')[1]} >>"
+        for i, ne in enumerate(named_entities)
+    ]
+
     array = [
         [""] + forms,
         ["*LEMMAS:*"] + lemmas,
@@ -124,6 +142,7 @@ def render_table(forms, lemmas, upos, xpos, feats, metadata, edges, edge_labels)
         ["*XPOS:*"] + xpos,
         ["*UFEATS:*"] + list(feats[0]),
         *([""] + list(row) for row in feats[1:])
+        ["*NE:*"] + named_entities,
     ]
 
     #return tabulate.tabulate(array, headers="firstrow", tablefmt="unsafehtml")
@@ -141,13 +160,13 @@ with gr.Blocks(theme='sudeepshouche/minimalist', css=custom_css) as demo:
     gr.HTML(description)
 
     with gr.Row():
-        with gr.Column(scale=1):
+        with gr.Column(scale=1, variant="panel"):
             source = gr.Textbox(
                 label="Input sentence", placeholder="Write a sentende to parse", show_label=False, lines=1, max_lines=5, autofocus=True
             )
             submit = gr.Button("Submit", variant="primary")
 
-        with gr.Column(scale=1):
+        with gr.Column(scale=1, variant="panel"):
             dataset = gr.Dataset(components=[gr.Textbox(visible=False)],
                 label="Input examples",
                 samples=[
@@ -161,4 +180,18 @@ with gr.Blocks(theme='sudeepshouche/minimalist', css=custom_css) as demo:
     table = gr.DataFrame(**render_table(forms, lemmas, upos, xpos, feats, metadata, edges, edge_labels), interactive=False, datatype="markdown")
     dependency_plot = gr.Plot(render_dependency_tree(forms, edges, edge_labels), container=False)
 
+    source.submit(
+        fn=parse, inputs=["source"], outputs=["dependency_plot", "table"], queue=True
+    )
+    submit.click(
+        fn=parse, inputs=["source"], outputs=["dependency_plot", "table"], queue=True
+    )
+    dataset.click(
+        fn=lambda text: text, inputs=["dataset"], outputs=["source"]
+    ).then(
+        fn=parse, inputs=["source"], outputs=["dependency_plot", "table"], queue=True
+    )
+
+
+demo.queue(max_size=32, concurrency_count=2)
 demo.launch()

config.json

@@ -0,0 +1,27 @@
+{
+    "architectures": [
+      "NorbertForMaskedLM"
+    ],
+    "auto_map": {
+      "AutoConfig": "modeling_norbert.NorbertConfig",
+      "AutoModel": "modeling_norbert.NorbertModel",
+      "AutoModelForMaskedLM": "modeling_norbert.NorbertForMaskedLM",
+      "AutoModelForSequenceClassification": "modeling_norbert.NorbertForSequenceClassification",
+      "AutoModelForTokenClassification": "modeling_norbert.NorbertForTokenClassification",
+      "AutoModelForQuestionAnswering": "modeling_norbert.NorbertForQuestionAnswering",
+      "AutoModelForMultipleChoice": "modeling_norbert.NorbertForMultipleChoice"
+    },
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 1024,
+    "intermediate_size": 2730,
+    "layer_norm_eps": 1e-07,
+    "max_position_embeddings": 512,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "position_bucket_size": 32,
+    "torch_dtype": "float32",
+    "transformers_version": "4.23.1",
+    "vocab_size": 50000
+  }
+  

dataset.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer
+
+from tokenizer import NLTKWordTokenizer
+from lemma_rule import apply_lemma_rule
+
+
+class Dataset:
+    def __init__(self):
+        self.word_tokenizer = NLTKWordTokenizer()
+        self.subword_tokenizer = AutoTokenizer.from_pretrained("ltg/norbert3-large")
+
+    def prepare_input(self, sentence: str):
+        word_spans = list(self.word_tokenizer.span_tokenize(sentence))
+        forms = [sentence[start:end] for start, end in word_spans]
+
+        subwords, alignment = [self.subword_tokenizer.convert_tokens_to_ids("[CLS]")], [0]
+        for i, word in enumerate(forms):
+            space_before = (i == 0) or sentence[word_spans[i - 1][1]] == " "
+
+            # very very ugly hack ;(
+            encoding = self.subword_tokenizer(f"| {word}" if space_before else f"|{word}", add_special_tokens=False)
+            subwords += encoding.input_ids[1:]
+            alignment += (len(encoding.input_ids) - 1) * [i + 1]
+
+        subwords.append(self.subword_tokenizer.convert_tokens_to_ids("[SEP]"))
+        alignment.append(alignment[-1] + 1)
+
+        subwords = torch.tensor([subwords])
+        alignment = torch.tensor([alignment])
+        alignment = F.one_hot(alignment, num_classes=len(forms) + 2).float()
+
+        return forms, subwords, alignment
+    
+    def decode_output(self, forms, lemma_p, upos_p, xpos_p, feats_p, dep_p, ne_p, head_p):
+        lemmas = [apply_lemma_rule(form, self.lemma_vocab[lemma_p[0, i, :].argmax().item()]) for i, form in enumerate(forms)]
+        upos = [self.upos_vocab[upos_p[0, i, :].argmax().item()] for i in range(len(forms))]
+        xpos = [self.xpos_vocab[xpos_p[0, i, :].argmax().item()] for i in range(len(forms))]
+        feats = [self.feats_vocab[feats_p[0, i, :].argmax().item()] for i in range(len(forms))]
+        heads = [head_p[0, i].item() for i in range(len(forms))]
+        deprel = [self.arc_dep_vocab[dep_p[0, i, :].argmax().item()] for i in range(len(forms))]
+        ne = [self.ne_vocab[ne_p[0, i, :].argmax().item()] for i in range(len(forms))]
+
+        return lemmas, upos, xpos, feats, heads, deprel, ne
+
+    # save state dict
+    def state_dict(self):
+        return {
+            "forms_vocab": self.forms_vocab,
+            "lemma_vocab": self.lemma_vocab,
+            "upos_vocab": self.upos_vocab,
+            "xpos_vocab": self.xpos_vocab,
+            "feats_vocab": self.feats_vocab,
+            "arc_dep_vocab": self.arc_dep_vocab,
+            "ne_vocab": self.ne_vocab
+        }
+
+    # load state dict
+    def load_state_dict(self, state_dict):
+        self.forms_vocab = state_dict["forms_vocab"]
+        self.lemma_vocab = state_dict["lemma_vocab"]
+        self.upos_vocab = state_dict["upos_vocab"]
+        self.xpos_vocab = state_dict["xpos_vocab"]
+        self.feats_vocab = state_dict["feats_vocab"]
+        self.arc_dep_vocab = state_dict["arc_dep_vocab"]
+        self.ne_vocab = state_dict["ne_vocab"]
+
+        self.lemma_indexer = {i: n for n, i in enumerate(self.lemma_vocab)}
+        self.upos_indexer = {i: n for n, i in enumerate(self.upos_vocab)}
+        self.xpos_indexer = {i: n for n, i in enumerate(self.xpos_vocab)}
+        self.feats_indexer = {i: n for n, i in enumerate(self.feats_vocab)}
+        self.ne_indexer = {i: n for n, i in enumerate(self.ne_vocab)}
+        self.arc_dep_indexer = {i: n for n, i in enumerate(self.arc_dep_vocab)}

lemma_rule.py

@@ -0,0 +1,101 @@
+def min_edit_script(source, target, allow_copy):
+    a = [[(len(source) + len(target) + 1, None)] * (len(target) + 1) for _ in range(len(source) + 1)]
+    for i in range(0, len(source) + 1):
+        for j in range(0, len(target) + 1):
+            if i == 0 and j == 0:
+                a[i][j] = (0, "")
+            else:
+                if allow_copy and i and j and source[i - 1] == target[j - 1] and a[i-1][j-1][0] < a[i][j][0]:
+                    a[i][j] = (a[i-1][j-1][0], a[i-1][j-1][1] + "→")
+                if i and a[i-1][j][0] < a[i][j][0]:
+                    a[i][j] = (a[i-1][j][0] + 1, a[i-1][j][1] + "-")
+                if j and a[i][j-1][0] < a[i][j][0]:
+                    a[i][j] = (a[i][j-1][0] + 1, a[i][j-1][1] + "+" + target[j - 1])
+    return a[-1][-1][1]
+
+
+def gen_lemma_rule(form, lemma, allow_copy):
+    form = form.lower()
+
+    previous_case = -1
+    lemma_casing = ""
+    for i, c in enumerate(lemma):
+        case = "↑" if c.lower() != c else "↓"
+        if case != previous_case:
+            lemma_casing += "{}{}{}".format("¦" if lemma_casing else "", case, i if i <= len(lemma) // 2 else i - len(lemma))
+        previous_case = case
+    lemma = lemma.lower()
+
+    best, best_form, best_lemma = 0, 0, 0
+    for l in range(len(lemma)):
+        for f in range(len(form)):
+            cpl = 0
+            while f + cpl < len(form) and l + cpl < len(lemma) and form[f + cpl] == lemma[l + cpl]: cpl += 1
+            if cpl > best:
+                best = cpl
+                best_form = f
+                best_lemma = l
+
+    rule = lemma_casing + ";"
+    if not best:
+        rule += "a" + lemma
+    else:
+        rule += "d{}¦{}".format(
+            min_edit_script(form[:best_form], lemma[:best_lemma], allow_copy),
+            min_edit_script(form[best_form + best:], lemma[best_lemma + best:], allow_copy),
+        )
+    return rule
+
+
+def apply_lemma_rule(form, lemma_rule):
+    if lemma_rule == "<unk>":
+        return form
+
+    if ';' not in lemma_rule:
+        raise ValueError('lemma_rule %r for form %r missing semicolon' %(lemma_rule, form))
+
+    casing, rule = lemma_rule.split(";", 1)
+    if rule.startswith("a"):
+        lemma = rule[1:]
+    else:
+        form = form.lower()
+        rules, rule_sources = rule[1:].split("¦"), []
+        assert len(rules) == 2
+        for rule in rules:
+            source, i = 0, 0
+            while i < len(rule):
+                if rule[i] == "→" or rule[i] == "-":
+                    source += 1
+                else:
+                    assert rule[i] == "+"
+                    i += 1
+                i += 1
+            rule_sources.append(source)
+
+        try:
+            lemma, form_offset = "", 0
+            for i in range(2):
+                j, offset = 0, (0 if i == 0 else len(form) - rule_sources[1])
+                while j < len(rules[i]):
+                    if rules[i][j] == "→":
+                        lemma += form[offset]
+                        offset += 1
+                    elif rules[i][j] == "-":
+                        offset += 1
+                    else:
+                        assert(rules[i][j] == "+")
+                        lemma += rules[i][j + 1]
+                        j += 1
+                    j += 1
+                if i == 0:
+                    lemma += form[rule_sources[0] : len(form) - rule_sources[1]]
+        except:
+            lemma = form
+
+    for rule in casing.split("¦"):
+        if rule == "↓0": continue # The lemma is lowercased initially
+        if not rule: continue # Empty lemma might generate empty casing rule
+        case, offset = rule[0], int(rule[1:])
+        lemma = lemma[:offset] + (lemma[offset:].upper() if case == "↑" else lemma[offset:].lower())
+
+    return lemma

model.py

@@ -0,0 +1,660 @@
+import math
+from typing import List, Optional, Tuple, Union
+import dependency_decoding
+import ftfy
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils import checkpoint
+
+from transformers.modeling_utils import PreTrainedModel
+from transformers.activations import gelu_new
+from transformers.modeling_outputs import (
+    MaskedLMOutput,
+    MultipleChoiceModelOutput,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutput,
+    TokenClassifierOutput,
+    BaseModelOutput
+)
+from transformers.pytorch_utils import softmax_backward_data
+from transformers.configuration_utils import PretrainedConfig
+
+from dataset import Dataset
+
+
+class NorbertConfig(PretrainedConfig):
+    """Configuration class to store the configuration of a `NorbertModel`.
+    """
+    def __init__(
+        self,
+        vocab_size=50000,
+        attention_probs_dropout_prob=0.1,
+        hidden_dropout_prob=0.1,
+        hidden_size=768,
+        intermediate_size=2048,
+        max_position_embeddings=512,
+        position_bucket_size=32,
+        num_attention_heads=12,
+        num_hidden_layers=12,
+        layer_norm_eps=1.0e-7,
+        output_all_encoded_layers=True,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.output_all_encoded_layers = output_all_encoded_layers
+        self.position_bucket_size = position_bucket_size
+        self.layer_norm_eps = layer_norm_eps
+
+
+class Encoder(nn.Module):
+    def __init__(self, config, activation_checkpointing=False):
+        super().__init__()
+        self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.num_hidden_layers)])
+
+        for i, layer in enumerate(self.layers):
+            layer.mlp.mlp[1].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
+            layer.mlp.mlp[-2].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
+
+        self.activation_checkpointing = activation_checkpointing
+    
+    def forward(self, hidden_states, attention_mask, relative_embedding):
+        hidden_states, attention_probs = [hidden_states], []
+
+        for layer in self.layers:
+            if self.activation_checkpointing:
+                hidden_state, attention_p = checkpoint.checkpoint(layer, hidden_states[-1], attention_mask, relative_embedding)
+            else:
+                hidden_state, attention_p = layer(hidden_states[-1], attention_mask, relative_embedding)
+
+            hidden_states.append(hidden_state)
+            attention_probs.append(attention_p)
+
+        return hidden_states, attention_probs
+
+
+class MaskClassifier(nn.Module):
+    def __init__(self, config, subword_embedding):
+        super().__init__()
+        self.nonlinearity = nn.Sequential(
+            nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
+            nn.Linear(config.hidden_size, config.hidden_size),
+            nn.GELU(),
+            nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
+            nn.Dropout(config.hidden_dropout_prob),
+            nn.Linear(subword_embedding.size(1), subword_embedding.size(0))
+        )
+        self.initialize(config.hidden_size, subword_embedding)
+
+    def initialize(self, hidden_size, embedding):
+        std = math.sqrt(2.0 / (5.0 * hidden_size))
+        nn.init.trunc_normal_(self.nonlinearity[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        self.nonlinearity[-1].weight = embedding
+        self.nonlinearity[1].bias.data.zero_()
+        self.nonlinearity[-1].bias.data.zero_()
+
+    def forward(self, x, masked_lm_labels=None):
+        if masked_lm_labels is not None:
+            x = torch.index_select(x.flatten(0, 1), 0, torch.nonzero(masked_lm_labels.flatten() != -100).squeeze())
+        x = self.nonlinearity(x)
+        return x
+
+
+class EncoderLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.attention = Attention(config)
+        self.mlp = FeedForward(config)
+
+    def forward(self, x, padding_mask, relative_embedding):
+        attention_output, attention_probs = self.attention(x, padding_mask, relative_embedding)
+        x = x + attention_output
+        x = x + self.mlp(x)
+        return x, attention_probs
+
+
+class GeGLU(nn.Module):
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        x = x * gelu_new(gate)
+        return x
+
+
+class FeedForward(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.mlp = nn.Sequential(
+            nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False),
+            nn.Linear(config.hidden_size, 2*config.intermediate_size, bias=False),
+            GeGLU(),
+            nn.LayerNorm(config.intermediate_size, eps=config.layer_norm_eps, elementwise_affine=False),
+            nn.Linear(config.intermediate_size, config.hidden_size, bias=False),
+            nn.Dropout(config.hidden_dropout_prob)
+        )
+        self.initialize(config.hidden_size)
+
+    def initialize(self, hidden_size):
+        std = math.sqrt(2.0 / (5.0 * hidden_size))
+        nn.init.trunc_normal_(self.mlp[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.mlp[-2].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+
+    def forward(self, x):
+        return self.mlp(x)
+
+
+class MaskedSoftmax(torch.autograd.Function):
+    @staticmethod
+    def forward(self, x, mask, dim):
+        self.dim = dim
+        x.masked_fill_(mask, float('-inf'))
+        x = torch.softmax(x, self.dim)
+        x.masked_fill_(mask, 0.0)
+        self.save_for_backward(x)
+        return x
+
+    @staticmethod
+    def backward(self, grad_output):
+        output, = self.saved_tensors
+        input_grad = softmax_backward_data(self, grad_output, output, self.dim, output)
+        return input_grad, None, None
+
+
+class Attention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+
+        self.config = config
+
+        if config.hidden_size % config.num_attention_heads != 0:
+            raise ValueError(f"The hidden size {config.hidden_size} is not a multiple of the number of attention heads {config.num_attention_heads}")
+
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_size = config.hidden_size // config.num_attention_heads
+
+        self.in_proj_qk = nn.Linear(config.hidden_size, 2*config.hidden_size, bias=True)
+        self.in_proj_v = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
+        self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
+
+        self.pre_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False)
+        self.post_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=True)
+
+        position_indices = torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(1) \
+            - torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(0)
+        position_indices = self.make_log_bucket_position(position_indices, config.position_bucket_size, config.max_position_embeddings)
+        position_indices = config.position_bucket_size - 1 + position_indices
+        self.register_buffer("position_indices", position_indices, persistent=True)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+        self.scale = 1.0 / math.sqrt(3 * self.head_size)
+        self.initialize()
+
+    def make_log_bucket_position(self, relative_pos, bucket_size, max_position):
+        sign = torch.sign(relative_pos)
+        mid = bucket_size // 2
+        abs_pos = torch.where((relative_pos < mid) & (relative_pos > -mid), mid - 1, torch.abs(relative_pos).clamp(max=max_position - 1))
+        log_pos = torch.ceil(torch.log(abs_pos / mid) / math.log((max_position-1) / mid) * (mid - 1)).int() + mid
+        bucket_pos = torch.where(abs_pos <= mid, relative_pos, log_pos * sign).long()
+        return bucket_pos
+
+    def initialize(self):
+        std = math.sqrt(2.0 / (5.0 * self.hidden_size))
+        nn.init.trunc_normal_(self.in_proj_qk.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.in_proj_v.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.out_proj.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        self.in_proj_qk.bias.data.zero_()
+        self.in_proj_v.bias.data.zero_()
+        self.out_proj.bias.data.zero_()
+
+    def compute_attention_scores(self, hidden_states, relative_embedding):
+        key_len, batch_size, _ = hidden_states.size()
+        query_len = key_len
+
+        if self.position_indices.size(0) < query_len:
+            position_indices = torch.arange(query_len, dtype=torch.long).unsqueeze(1) \
+                - torch.arange(query_len, dtype=torch.long).unsqueeze(0)
+            position_indices = self.make_log_bucket_position(position_indices, self.position_bucket_size, 512)
+            position_indices = self.position_bucket_size - 1 + position_indices
+            self.position_indices = position_indices.to(hidden_states.device)
+
+        hidden_states = self.pre_layer_norm(hidden_states)
+
+        query, key = self.in_proj_qk(hidden_states).chunk(2, dim=2)  # shape: [T, B, D]
+        value = self.in_proj_v(hidden_states)  # shape: [T, B, D]
+
+        query = query.reshape(query_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
+        key = key.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
+        value = value.view(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
+
+        attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)
+
+        pos = self.in_proj_qk(self.dropout(relative_embedding))  # shape: [2T-1, 2D]
+        query_pos, key_pos = pos.view(-1, self.num_heads, 2*self.head_size).chunk(2, dim=2)
+        query = query.view(batch_size, self.num_heads, query_len, self.head_size)
+        key = key.view(batch_size, self.num_heads, query_len, self.head_size)
+
+        attention_c_p = torch.einsum("bhqd,khd->bhqk", query, key_pos.squeeze(1) * self.scale)
+        attention_p_c = torch.einsum("bhkd,qhd->bhqk", key * self.scale, query_pos.squeeze(1))
+
+        position_indices = self.position_indices[:query_len, :key_len].expand(batch_size, self.num_heads, -1, -1)
+        attention_c_p = attention_c_p.gather(3, position_indices)
+        attention_p_c = attention_p_c.gather(2, position_indices)
+
+        attention_scores = attention_scores.view(batch_size, self.num_heads, query_len, key_len)
+        attention_scores.add_(attention_c_p)
+        attention_scores.add_(attention_p_c)
+
+        return attention_scores, value
+
+    def compute_output(self, attention_probs, value):
+        attention_probs = self.dropout(attention_probs)
+        context = torch.bmm(attention_probs.flatten(0, 1), value)  # shape: [B*H, Q, D]
+        context = context.transpose(0, 1).reshape(context.size(1), -1, self.hidden_size)  # shape: [Q, B, H*D]
+        context = self.out_proj(context)
+        context = self.post_layer_norm(context)
+        context = self.dropout(context)
+        return context
+
+    def forward(self, hidden_states, attention_mask, relative_embedding):
+        attention_scores, value = self.compute_attention_scores(hidden_states, relative_embedding)
+        attention_probs = MaskedSoftmax.apply(attention_scores, attention_mask, -1)
+        return self.compute_output(attention_probs, value), attention_probs.detach()
+
+
+class Embedding(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+
+        self.word_embedding = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.word_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+        self.relative_embedding = nn.Parameter(torch.empty(2 * config.position_bucket_size - 1, config.hidden_size))
+        self.relative_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+        self.initialize()
+
+    def initialize(self):
+        std = math.sqrt(2.0 / (5.0 * self.hidden_size))
+        nn.init.trunc_normal_(self.relative_embedding, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.word_embedding.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+
+    def forward(self, input_ids):
+        word_embedding = self.dropout(self.word_layer_norm(self.word_embedding(input_ids)))
+        relative_embeddings = self.relative_layer_norm(self.relative_embedding)
+        return word_embedding, relative_embeddings
+
+
+#
+# HuggingFace wrappers
+#
+
+class NorbertPreTrainedModel(PreTrainedModel):
+    config_class = NorbertConfig
+    base_model_prefix = "norbert3"
+    supports_gradient_checkpointing = True
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, Encoder):
+            module.activation_checkpointing = value
+
+    def _init_weights(self, module):
+        pass  # everything is already initialized
+
+
+class NorbertModel(NorbertPreTrainedModel):
+    def __init__(self, config, add_mlm_layer=False, gradient_checkpointing=False, **kwargs):
+        super().__init__(config, **kwargs)
+        self.config = config
+
+        self.embedding = Embedding(config)
+        self.transformer = Encoder(config, activation_checkpointing=gradient_checkpointing)
+        self.classifier = MaskClassifier(config, self.embedding.word_embedding.weight) if add_mlm_layer else None
+
+    def get_input_embeddings(self):
+        return self.embedding.word_embedding
+
+    def set_input_embeddings(self, value):
+        self.embedding.word_embedding = value
+
+    def get_contextualized_embeddings(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None
+    ) -> List[torch.Tensor]:
+        if input_ids is not None:
+            input_shape = input_ids.size()
+        else:
+            raise ValueError("You have to specify input_ids")
+
+        batch_size, seq_length = input_shape
+        device = input_ids.device
+
+        if attention_mask is None:
+            attention_mask = torch.zeros(batch_size, seq_length, dtype=torch.bool, device=device)
+        else:
+            attention_mask = ~attention_mask.bool()
+        attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+ 
+        static_embeddings, relative_embedding = self.embedding(input_ids.t())
+        contextualized_embeddings, attention_probs = self.transformer(static_embeddings, attention_mask, relative_embedding)
+        contextualized_embeddings = [e.transpose(0, 1) for e in contextualized_embeddings]
+        last_layer = contextualized_embeddings[-1]
+        contextualized_embeddings = [contextualized_embeddings[0]] + [
+            contextualized_embeddings[i] - contextualized_embeddings[i - 1]
+            for i in range(1, len(contextualized_embeddings))
+        ]
+        return last_layer, contextualized_embeddings, attention_probs
+
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        output_hidden_states: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs
+    ) -> Union[Tuple[torch.Tensor], BaseModelOutput]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+
+        if not return_dict:
+            return (
+                sequence_output,
+                *([contextualized_embeddings] if output_hidden_states else []),
+                *([attention_probs] if output_attentions else [])
+            )
+
+        return BaseModelOutput(
+            last_hidden_state=sequence_output,
+            hidden_states=contextualized_embeddings if output_hidden_states else None,
+            attentions=attention_probs if output_attentions else None
+        )
+
+
+class Classifier(nn.Module):
+    def __init__(self, hidden_size, vocab_size, dropout):
+        super().__init__()
+
+        self.transform = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.GELU(),
+            nn.LayerNorm(hidden_size, elementwise_affine=False),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_size, vocab_size)
+        )
+        self.initialize(hidden_size)
+
+    def initialize(self, hidden_size):
+        std = math.sqrt(2.0 / (5.0 * hidden_size))
+        nn.init.trunc_normal_(self.transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.transform[-1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        self.transform[0].bias.data.zero_()
+        self.transform[-1].bias.data.zero_()
+
+    def forward(self, x):
+        return self.transform(x)
+
+
+class ZeroClassifier(nn.Module):
+    def forward(self, x):
+        output = torch.zeros(x.size(0), x.size(1), 2, device=x.device, dtype=x.dtype)
+        output[:, :, 0] = 1.0
+        output[:, :, 1] = -1.0
+        return output
+
+
+class EdgeClassifier(nn.Module):
+    def __init__(self, hidden_size, dep_hidden_size, vocab_size, dropout):
+        super().__init__()
+
+        self.head_dep_transform = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.GELU(),
+            nn.LayerNorm(hidden_size, elementwise_affine=False),
+            nn.Dropout(dropout)
+        )
+        self.head_root_transform = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.GELU(),
+            nn.LayerNorm(hidden_size, elementwise_affine=False),
+            nn.Dropout(dropout)
+        )
+        self.head_bilinear = nn.Parameter(torch.zeros(hidden_size, hidden_size))
+        self.head_linear_dep = nn.Linear(hidden_size, 1, bias=False)
+        self.head_linear_root = nn.Linear(hidden_size, 1, bias=False)
+        self.head_bias = nn.Parameter(torch.zeros(1))
+
+        self.dep_dep_transform = nn.Sequential(
+            nn.Linear(hidden_size, dep_hidden_size),
+            nn.GELU(),
+            nn.LayerNorm(dep_hidden_size, elementwise_affine=False),
+            nn.Dropout(dropout)
+        )
+        self.dep_root_transform = nn.Sequential(
+            nn.Linear(hidden_size, dep_hidden_size),
+            nn.GELU(),
+            nn.LayerNorm(dep_hidden_size, elementwise_affine=False),
+            nn.Dropout(dropout)
+        )
+        self.dep_bilinear = nn.Parameter(torch.zeros(dep_hidden_size, dep_hidden_size, vocab_size))
+        self.dep_linear_dep = nn.Linear(dep_hidden_size, vocab_size, bias=False)
+        self.dep_linear_root = nn.Linear(dep_hidden_size, vocab_size, bias=False)
+        self.dep_bias = nn.Parameter(torch.zeros(vocab_size))
+
+        self.hidden_size = hidden_size
+        self.dep_hidden_size = dep_hidden_size
+
+        self.mask_value = float("-inf")
+        self.initialize(hidden_size)
+
+    def initialize(self, hidden_size):
+        std = math.sqrt(2.0 / (5.0 * hidden_size))
+        nn.init.trunc_normal_(self.head_dep_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.head_root_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.dep_dep_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.dep_root_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+
+        nn.init.trunc_normal_(self.head_linear_dep.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.head_linear_root.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.dep_linear_dep.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.dep_linear_root.weight, mean=0.0, std=std, a=-2*std, b=2*std)
+
+        self.head_dep_transform[0].bias.data.zero_()
+        self.head_root_transform[0].bias.data.zero_()
+        self.dep_dep_transform[0].bias.data.zero_()
+        self.dep_root_transform[0].bias.data.zero_()
+
+    def forward(self, head_x, dep_x, lengths, head_gold=None):
+        head_dep = self.head_dep_transform(head_x[:, 1:, :])
+        head_root = self.head_root_transform(head_x)
+        head_prediction = torch.einsum("bkn,nm,blm->bkl", head_dep, self.head_bilinear, head_root / math.sqrt(self.hidden_size)) \
+            + self.head_linear_dep(head_dep) + self.head_linear_root(head_root).transpose(1, 2) + self.head_bias
+
+        mask = (torch.arange(head_x.size(1)).unsqueeze(0) >= lengths.unsqueeze(1)).unsqueeze(1).to(head_x.device)
+        mask = mask | (torch.ones(head_x.size(1) - 1, head_x.size(1), dtype=torch.bool, device=head_x.device).tril(1) & torch.ones(head_x.size(1) - 1, head_x.size(1), dtype=torch.bool, device=head_x.device).triu(1))
+        head_prediction = head_prediction.masked_fill(mask, self.mask_value)
+
+        if head_gold is None:
+            head_logp = torch.log_softmax(head_prediction, dim=-1)
+            head_logp = F.pad(head_logp, (0, 0, 1, 0), value=torch.nan).cpu()
+            head_gold = []
+            for i, length in enumerate(lengths.tolist()):
+                head = self.max_spanning_tree(head_logp[i, :length, :length])
+                head = head + ((head_x.size(1) - 1) - len(head)) * [0]
+                head_gold.append(torch.tensor(head))
+            head_gold = torch.stack(head_gold).to(head_x.device)
+
+        dep_dep = self.dep_dep_transform(dep_x[:, 1:])
+        dep_root = dep_x.gather(1, head_gold.unsqueeze(-1).expand(-1, -1, dep_x.size(-1)).clamp(min=0))
+        dep_root = self.dep_root_transform(dep_root)
+        dep_prediction = torch.einsum("btm,mnl,btn->btl", dep_dep, self.dep_bilinear, dep_root / math.sqrt(self.dep_hidden_size)) \
+            + self.dep_linear_dep(dep_dep) + self.dep_linear_root(dep_root) + self.dep_bias
+
+        return head_prediction, dep_prediction, head_gold
+    
+    def max_spanning_tree(self, weight_matrix):
+        weight_matrix = weight_matrix.clone()
+        # weight_matrix[:, 0] = torch.nan
+
+        # we need to make sure that the root is the parent of a single node
+        # first, we try to use the default weights, it should work in most cases
+        parents, _ = dependency_decoding.chu_liu_edmonds(weight_matrix.numpy().astype(float))
+
+        assert parents[0] == -1, f"{parents}\n{weight_matrix}"
+        parents = parents[1:]
+
+        # check if the root is the parent of a single node
+        if parents.count(0) == 1:
+            return parents
+        
+        # if not, we need to modify the weights and try all possibilities
+        # we try to find the node that is the parent of the root
+        best_score = float("-inf")
+        best_parents = None
+
+        for i in range(len(parents)):
+            weight_matrix_mod = weight_matrix.clone()
+            weight_matrix_mod[:i+1, 0] = torch.nan
+            weight_matrix_mod[i+2:, 0] = torch.nan
+            parents, score = dependency_decoding.chu_liu_edmonds(weight_matrix_mod.numpy().astype(float))
+            parents = parents[1:]
+
+            if score > best_score:
+                best_score = score
+                best_parents = parents
+
+        def print_whole_matrix(matrix):
+            for i in range(matrix.shape[0]):
+                print(" ".join([str(x) for x in matrix[i]]))
+
+        assert best_parents is not None, f"{best_parents}\n{print_whole_matrix(weight_matrix)}"
+        return best_parents
+
+
+class Model(nn.Module):
+    def __init__(self, dataset):
+        super().__init__()
+
+#        config = BertConfig("../../configs/base.json")
+#        self.bert = Bert(config)
+#        checkpoint = torch.load("../../checkpoints/test_wd=0.01/model.bin", map_location="cpu")
+#        self.bert.load_state_dict(checkpoint["model"], strict=False)
+
+        config = NorbertConfig.from_json_file("config.json")
+        self.bert = NorbertModel(config)
+
+        self.n_layers = config.num_hidden_layers
+
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False)
+        self.upos_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
+        self.xpos_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
+        self.feats_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
+        self.lemma_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
+        self.head_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
+        self.dep_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
+        self.ner_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
+
+        self.lemma_classifier = Classifier(config.hidden_size, len(dataset.lemma_vocab), config.hidden_dropout_prob)
+        self.upos_classifier = Classifier(config.hidden_size, len(dataset.upos_vocab), config.hidden_dropout_prob) if len(dataset.upos_vocab) > 2 else ZeroClassifier()
+        self.xpos_classifier = Classifier(config.hidden_size, len(dataset.xpos_vocab), config.hidden_dropout_prob) if len(dataset.xpos_vocab) > 2 else ZeroClassifier()
+        self.feats_classifier = Classifier(config.hidden_size, len(dataset.feats_vocab), config.hidden_dropout_prob) if len(dataset.feats_vocab) > 2 else ZeroClassifier()
+        self.edge_classifier = EdgeClassifier(config.hidden_size, 128, len(dataset.arc_dep_vocab), config.hidden_dropout_prob)
+        self.ner_classifier = Classifier(config.hidden_size, len(dataset.ne_vocab), config.hidden_dropout_prob) if len(dataset.ne_vocab) > 2 else ZeroClassifier()
+
+    def forward(self, x, alignment_mask, subword_lengths, word_lengths, head_gold=None):
+        padding_mask = (torch.arange(x.size(1)).unsqueeze(0) < subword_lengths.unsqueeze(1)).to(x.device)
+        x = self.bert(x, padding_mask, output_hidden_states=True).hidden_states
+        x = torch.stack(x, dim=0)
+
+        upos_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.upos_layer_score, dim=0))
+        xpos_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.xpos_layer_score, dim=0))
+        feats_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.feats_layer_score, dim=0))
+        lemma_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.lemma_layer_score, dim=0))
+        head_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.head_layer_score, dim=0))
+        dep_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.dep_layer_score, dim=0))
+        ne_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.ner_layer_score, dim=0))
+
+        upos_x = torch.einsum("bsd,bst->btd", upos_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
+        xpos_x = torch.einsum("bsd,bst->btd", xpos_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
+        feats_x = torch.einsum("bsd,bst->btd", feats_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
+        lemma_x = torch.einsum("bsd,bst->btd", lemma_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
+        head_x = torch.einsum("bsd,bst->btd", head_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
+        dep_x = torch.einsum("bsd,bst->btd", dep_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
+        ne_x = torch.einsum("bsd, bst -> btd", ne_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
+
+        upos_x = self.dropout(self.layer_norm(upos_x[:, 1:-1, :]))
+        xpos_x = self.dropout(self.layer_norm(xpos_x[:, 1:-1, :]))
+        feats_x = self.dropout(self.layer_norm(feats_x[:, 1:-1, :]))
+        lemma_x = self.dropout(self.layer_norm(lemma_x[:, 1:-1, :]))
+        head_x = self.dropout(self.layer_norm(head_x[:, 0:-1, :]))
+        dep_x = self.dropout(self.layer_norm(dep_x[:, 0:-1, :]))
+        ne_x = self.dropout(self.layer_norm(ne_x[:, 1:-1, :]))
+
+        lemma_preds = self.lemma_classifier(lemma_x)
+        upos_preds = self.upos_classifier(upos_x)
+        xpos_preds = self.xpos_classifier(xpos_x)
+        feats_preds = self.feats_classifier(feats_x)
+        ne_preds = self.ner_classifier(feats_x)
+        head_prediction, dep_prediction, head_liu = self.edge_classifier(head_x, dep_x, word_lengths, head_gold)
+
+        return lemma_preds, upos_preds, xpos_preds, feats_preds, head_prediction, dep_prediction, ne_preds, head_liu
+
+
+class Parser:
+    def __init__(self):
+        checkpoint = torch.load("checkpoint.bin", map_location="cpu")
+
+        self.dataset = Dataset()
+        self.dataset.load_state_dict(checkpoint["dataset"])
+
+        self.model = Model(self.dataset)
+        self.model.load_state_dict(checkpoint["model"])
+        self.model.eval()
+        del checkpoint
+
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model.to(self.device)
+
+    def parse(self, sentence):
+        sentence = ftfy.fix_text(sentence.strip())
+        forms, subwords, alignment = self.dataset.prepare_input(sentence)
+
+        with torch.no_grad():
+            output = self.model(
+                subwords.to(self.device),
+                alignment.to(self.device),
+                torch.tensor([len(forms) + 1], device=self.device),
+                torch.tensor([subwords.size(1)], device=self.device)
+            )
+ 
+        lemma_p, upos_p, xpos_p, feats_p, _, dep_p, ne_p, head_p = output
+        lemmas, upos, xpos, feats, heads, deprel, ne = self.dataset.decode_output(
+            forms, lemma_p, upos_p, xpos_p, feats_p, dep_p, ne_p, head_p
+        )
+
+        return {
+            "forms": forms,
+            "lemmas": lemmas,
+            "upos": upos,
+            "xpos": xpos,
+            "feats": feats,
+            "heads": heads,
+            "deprel": deprel,
+            "ne": ne
+        }

requirements.txt

@@ -1,4 +1,8 @@
 tabulate
 matplotlib
 networkx
-pygraphviz
+pygraphviz
+ftfy
+torch
+transformers
+dependency_decoding

tokenizer.py

@@ -0,0 +1,231 @@
+# Natural Language Toolkit: NLTK's very own tokenizer, slightly modified.
+#
+# Copyright (C) 2001-2023 NLTK Project
+# Author: Liling Tan
+#         Tom Aarsen <> (modifications)
+# URL: <https://www.nltk.org>
+
+
+import re
+import warnings
+from typing import Iterator, List, Tuple
+
+
+def align_tokens(tokens, sentence):
+    """
+    This module attempt to find the offsets of the tokens in *s*, as a sequence
+    of ``(start, end)`` tuples, given the tokens and also the source string.
+
+        >>> from nltk.tokenize import TreebankWordTokenizer
+        >>> from nltk.tokenize.util import align_tokens
+        >>> s = str("The plane, bound for St Petersburg, crashed in Egypt's "
+        ... "Sinai desert just 23 minutes after take-off from Sharm el-Sheikh "
+        ... "on Saturday.")
+        >>> tokens = TreebankWordTokenizer().tokenize(s)
+        >>> expected = [(0, 3), (4, 9), (9, 10), (11, 16), (17, 20), (21, 23),
+        ... (24, 34), (34, 35), (36, 43), (44, 46), (47, 52), (52, 54),
+        ... (55, 60), (61, 67), (68, 72), (73, 75), (76, 83), (84, 89),
+        ... (90, 98), (99, 103), (104, 109), (110, 119), (120, 122),
+        ... (123, 131), (131, 132)]
+        >>> output = list(align_tokens(tokens, s))
+        >>> len(tokens) == len(expected) == len(output)  # Check that length of tokens and tuples are the same.
+        True
+        >>> expected == list(align_tokens(tokens, s))  # Check that the output is as expected.
+        True
+        >>> tokens == [s[start:end] for start, end in output]  # Check that the slices of the string corresponds to the tokens.
+        True
+
+    :param tokens: The list of strings that are the result of tokenization
+    :type tokens: list(str)
+    :param sentence: The original string
+    :type sentence: str
+    :rtype: list(tuple(int,int))
+    """
+    point = 0
+    offsets = []
+    for token in tokens:
+        try:
+            start = sentence.index(token, point)
+        except ValueError as e:
+            raise ValueError(f'substring "{token}" not found in "{sentence}"') from e
+        point = start + len(token)
+        offsets.append((start, point))
+    return offsets
+
+
+class NLTKWordTokenizer:
+    """
+    The NLTK tokenizer that has improved upon the TreebankWordTokenizer.
+
+    This is the method that is invoked by ``word_tokenize()``.  It assumes that the
+    text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
+
+    The tokenizer is "destructive" such that the regexes applied will munge the
+    input string to a state beyond re-construction. It is possible to apply
+    `TreebankWordDetokenizer.detokenize` to the tokenized outputs of
+    `NLTKDestructiveWordTokenizer.tokenize` but there's no guarantees to
+    revert to the original string.
+    """
+
+    # Starting quotes.
+    STARTING_QUOTES = [
+        (re.compile("([«“‘„]|[`]+)", re.U), r" \1 "),
+        (re.compile(r"^\""), r' " '),
+        (re.compile(r"(``)"), r" \1 "),
+        (re.compile(r"([ \(\[{<])(\"|\'{2})"), r'\1 " '),
+#        (re.compile(r"(?i)(\')(?!re|ve|ll|m|t|s|d|n)(\w)\b", re.U), r"\1 \2"),
+    ]
+
+    # Ending quotes.
+    ENDING_QUOTES = [
+        (re.compile("([»”’])", re.U), r" \1 "),
+        (re.compile(r"''"), " '' "),
+        (re.compile(r'"'), ' " '),
+        (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
+#        (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
+    ]
+
+    # For improvements for starting/closing quotes from TreebankWordTokenizer,
+    # see discussion on https://github.com/nltk/nltk/pull/1437
+    # Adding to TreebankWordTokenizer, nltk.word_tokenize now splits on
+    # - chervon quotes u'\xab' and u'\xbb' .
+    # - unicode quotes u'\u2018', u'\u2019', u'\u201c' and u'\u201d'
+    # See https://github.com/nltk/nltk/issues/1995#issuecomment-376741608
+    # Also, behavior of splitting on clitics now follows Stanford CoreNLP
+    # - clitics covered (?!re|ve|ll|m|t|s|d)(\w)\b
+
+    # Punctuation.
+    PUNCTUATION = [
+        (re.compile(r'([^\.])(\.)([\]\)}>"\'' "»”’ " r"]*)\s*$", re.U), r"\1 \2 \3 "),
+        (re.compile(r"([:,])([^\d])"), r" \1 \2"),
+        (re.compile(r"([:,])$"), r" \1 "),
+        (
+            re.compile(r"\.{2,}", re.U),
+            r" \g<0> ",
+        ),  # See https://github.com/nltk/nltk/pull/2322
+        (re.compile(r"[;@#$%&]"), r" \g<0> "),
+        (
+            re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'),
+            r"\1 \2\3 ",
+        ),  # Handles the final period.
+        (re.compile(r"[?!]"), r" \g<0> "),
+        (re.compile(r"([^'])' "), r"\1 ' "),
+        (
+            re.compile(r"[*]", re.U),
+            r" \g<0> ",
+        ),  # See https://github.com/nltk/nltk/pull/2322
+    ]
+
+    # Pads parentheses
+    PARENS_BRACKETS = (re.compile(r"[\]\[\(\)\{\}\<\>]"), r" \g<0> ")
+
+    # Optionally: Convert parentheses, brackets and converts them to PTB symbols.
+    # CONVERT_PARENTHESES = [
+    #     (re.compile(r"\("), "-LRB-"),
+    #     (re.compile(r"\)"), "-RRB-"),
+    #     (re.compile(r"\["), "-LSB-"),
+    #     (re.compile(r"\]"), "-RSB-"),
+    #     (re.compile(r"\{"), "-LCB-"),
+    #     (re.compile(r"\}"), "-RCB-"),
+    # ]
+
+    DOUBLE_DASHES = (re.compile(r"--"), r" -- ")
+
+    # List of contractions adapted from Robert MacIntyre's tokenizer.
+    # _contractions = MacIntyreContractions()
+    # CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
+    # CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
+
+    def tokenize(
+        self, text: str
+    ) -> List[str]:
+        r"""Return a tokenized copy of `text`.
+
+        >>> from nltk.tokenize import NLTKWordTokenizer
+        >>> s = '''Good muffins cost $3.88 (roughly 3,36 euros)\nin New York.  Please buy me\ntwo of them.\nThanks.'''
+        >>> NLTKWordTokenizer().tokenize(s) # doctest: +NORMALIZE_WHITESPACE
+        ['Good', 'muffins', 'cost', '$', '3.88', '(', 'roughly', '3,36',
+        'euros', ')', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+        >>> NLTKWordTokenizer().tokenize(s, convert_parentheses=True) # doctest: +NORMALIZE_WHITESPACE
+        ['Good', 'muffins', 'cost', '$', '3.88', '-LRB-', 'roughly', '3,36',
+        'euros', '-RRB-', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two',
+        'of', 'them.', 'Thanks', '.']
+
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :param convert_parentheses: if True, replace parentheses to PTB symbols,
+            e.g. `(` to `-LRB-`. Defaults to False.
+        :type convert_parentheses: bool, optional
+        :param return_str: If True, return tokens as space-separated string,
+            defaults to False.
+        :type return_str: bool, optional
+        :return: List of tokens from `text`.
+        :rtype: List[str]
+        """
+
+        for regexp, substitution in self.STARTING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        for regexp, substitution in self.PUNCTUATION:
+            text = regexp.sub(substitution, text)
+
+        # Handles parentheses.
+        regexp, substitution = self.PARENS_BRACKETS
+        text = regexp.sub(substitution, text)
+
+        # Handles double dash.
+        regexp, substitution = self.DOUBLE_DASHES
+        text = regexp.sub(substitution, text)
+
+        # add extra space to make things easier
+        text = " " + text + " "
+
+        for regexp, substitution in self.ENDING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        return text.split()
+
+    def span_tokenize(self, text: str) -> Iterator[Tuple[int, int]]:
+        r"""
+        Returns the spans of the tokens in ``text``.
+        Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
+
+            >>> from nltk.tokenize import NLTKWordTokenizer
+            >>> s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
+            >>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
+            ... (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
+            ... (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
+            ... (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+            >>> list(NLTKWordTokenizer().span_tokenize(s)) == expected
+            True
+            >>> expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+            ... 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
+            ... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+            >>> [s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected
+            True
+
+        :param text: A string with a sentence or sentences.
+        :type text: str
+        :yield: Tuple[int, int]
+        """
+        raw_tokens = self.tokenize(text)
+
+        # Convert converted quotes back to original double quotes
+        # Do this only if original text contains double quote(s) or double
+        # single-quotes (because '' might be transformed to `` if it is
+        # treated as starting quotes).
+        # if ('"' in text) or ("''" in text):
+        #     # Find double quotes and converted quotes
+        #     matched = [m.group() for m in re.finditer(r"``|'{2}|\"", text)]
+
+        #     # Replace converted quotes back to double quotes
+        #     tokens = [
+        #         matched.pop(0) if tok in ['"', "``", "''"] else tok
+        #         for tok in raw_tokens
+        #     ]
+        # else:
+        tokens = raw_tokens
+
+        yield from align_tokens(tokens, text)