model.py

import math
from typing import List, Optional, Tuple, Union
import dependency_decoding
import ftfy

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import checkpoint

from transformers.modeling_utils import PreTrainedModel
from transformers.activations import gelu_new
from transformers.modeling_outputs import (
    MaskedLMOutput,
    MultipleChoiceModelOutput,
    QuestionAnsweringModelOutput,
    SequenceClassifierOutput,
    TokenClassifierOutput,
    BaseModelOutput
)
from transformers.pytorch_utils import softmax_backward_data
from transformers.configuration_utils import PretrainedConfig

from dataset import Dataset


class NorbertConfig(PretrainedConfig):
    """Configuration class to store the configuration of a `NorbertModel`.
    """
    def __init__(
        self,
        vocab_size=50000,
        attention_probs_dropout_prob=0.1,
        hidden_dropout_prob=0.1,
        hidden_size=768,
        intermediate_size=2048,
        max_position_embeddings=512,
        position_bucket_size=32,
        num_attention_heads=12,
        num_hidden_layers=12,
        layer_norm_eps=1.0e-7,
        output_all_encoded_layers=True,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.output_all_encoded_layers = output_all_encoded_layers
        self.position_bucket_size = position_bucket_size
        self.layer_norm_eps = layer_norm_eps


class Encoder(nn.Module):
    def __init__(self, config, activation_checkpointing=False):
        super().__init__()
        self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.num_hidden_layers)])

        for i, layer in enumerate(self.layers):
            layer.mlp.mlp[1].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
            layer.mlp.mlp[-2].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))

        self.activation_checkpointing = activation_checkpointing
    
    def forward(self, hidden_states, attention_mask, relative_embedding):
        hidden_states, attention_probs = [hidden_states], []

        for layer in self.layers:
            if self.activation_checkpointing:
                hidden_state, attention_p = checkpoint.checkpoint(layer, hidden_states[-1], attention_mask, relative_embedding)
            else:
                hidden_state, attention_p = layer(hidden_states[-1], attention_mask, relative_embedding)

            hidden_states.append(hidden_state)
            attention_probs.append(attention_p)

        return hidden_states, attention_probs


class MaskClassifier(nn.Module):
    def __init__(self, config, subword_embedding):
        super().__init__()
        self.nonlinearity = nn.Sequential(
            nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
            nn.Linear(config.hidden_size, config.hidden_size),
            nn.GELU(),
            nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
            nn.Dropout(config.hidden_dropout_prob),
            nn.Linear(subword_embedding.size(1), subword_embedding.size(0))
        )
        self.initialize(config.hidden_size, subword_embedding)

    def initialize(self, hidden_size, embedding):
        std = math.sqrt(2.0 / (5.0 * hidden_size))
        nn.init.trunc_normal_(self.nonlinearity[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
        self.nonlinearity[-1].weight = embedding
        self.nonlinearity[1].bias.data.zero_()
        self.nonlinearity[-1].bias.data.zero_()

    def forward(self, x, masked_lm_labels=None):
        if masked_lm_labels is not None:
            x = torch.index_select(x.flatten(0, 1), 0, torch.nonzero(masked_lm_labels.flatten() != -100).squeeze())
        x = self.nonlinearity(x)
        return x


class EncoderLayer(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.attention = Attention(config)
        self.mlp = FeedForward(config)

    def forward(self, x, padding_mask, relative_embedding):
        attention_output, attention_probs = self.attention(x, padding_mask, relative_embedding)
        x = x + attention_output
        x = x + self.mlp(x)
        return x, attention_probs


class GeGLU(nn.Module):
    def forward(self, x):
        x, gate = x.chunk(2, dim=-1)
        x = x * gelu_new(gate)
        return x


class FeedForward(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.mlp = nn.Sequential(
            nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False),
            nn.Linear(config.hidden_size, 2*config.intermediate_size, bias=False),
            GeGLU(),
            nn.LayerNorm(config.intermediate_size, eps=config.layer_norm_eps, elementwise_affine=False),
            nn.Linear(config.intermediate_size, config.hidden_size, bias=False),
            nn.Dropout(config.hidden_dropout_prob)
        )
        self.initialize(config.hidden_size)

    def initialize(self, hidden_size):
        std = math.sqrt(2.0 / (5.0 * hidden_size))
        nn.init.trunc_normal_(self.mlp[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.mlp[-2].weight, mean=0.0, std=std, a=-2*std, b=2*std)

    def forward(self, x):
        return self.mlp(x)


class MaskedSoftmax(torch.autograd.Function):
    @staticmethod
    def forward(self, x, mask, dim):
        self.dim = dim
        x.masked_fill_(mask, float('-inf'))
        x = torch.softmax(x, self.dim)
        x.masked_fill_(mask, 0.0)
        self.save_for_backward(x)
        return x

    @staticmethod
    def backward(self, grad_output):
        output, = self.saved_tensors
        input_grad = softmax_backward_data(self, grad_output, output, self.dim, output)
        return input_grad, None, None


class Attention(nn.Module):
    def __init__(self, config):
        super().__init__()

        self.config = config

        if config.hidden_size % config.num_attention_heads != 0:
            raise ValueError(f"The hidden size {config.hidden_size} is not a multiple of the number of attention heads {config.num_attention_heads}")

        self.hidden_size = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_size = config.hidden_size // config.num_attention_heads

        self.in_proj_qk = nn.Linear(config.hidden_size, 2*config.hidden_size, bias=True)
        self.in_proj_v = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
        self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)

        self.pre_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False)
        self.post_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=True)

        position_indices = torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(1) \
            - torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(0)
        position_indices = self.make_log_bucket_position(position_indices, config.position_bucket_size, config.max_position_embeddings)
        position_indices = config.position_bucket_size - 1 + position_indices
        self.register_buffer("position_indices", position_indices, persistent=True)

        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
        self.scale = 1.0 / math.sqrt(3 * self.head_size)
        self.initialize()

    def make_log_bucket_position(self, relative_pos, bucket_size, max_position):
        sign = torch.sign(relative_pos)
        mid = bucket_size // 2
        abs_pos = torch.where((relative_pos < mid) & (relative_pos > -mid), mid - 1, torch.abs(relative_pos).clamp(max=max_position - 1))
        log_pos = torch.ceil(torch.log(abs_pos / mid) / math.log((max_position-1) / mid) * (mid - 1)).int() + mid
        bucket_pos = torch.where(abs_pos <= mid, relative_pos, log_pos * sign).long()
        return bucket_pos

    def initialize(self):
        std = math.sqrt(2.0 / (5.0 * self.hidden_size))
        nn.init.trunc_normal_(self.in_proj_qk.weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.in_proj_v.weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.out_proj.weight, mean=0.0, std=std, a=-2*std, b=2*std)
        self.in_proj_qk.bias.data.zero_()
        self.in_proj_v.bias.data.zero_()
        self.out_proj.bias.data.zero_()

    def compute_attention_scores(self, hidden_states, relative_embedding):
        key_len, batch_size, _ = hidden_states.size()
        query_len = key_len

        if self.position_indices.size(0) < query_len:
            position_indices = torch.arange(query_len, dtype=torch.long).unsqueeze(1) \
                - torch.arange(query_len, dtype=torch.long).unsqueeze(0)
            position_indices = self.make_log_bucket_position(position_indices, self.position_bucket_size, 512)
            position_indices = self.position_bucket_size - 1 + position_indices
            self.position_indices = position_indices.to(hidden_states.device)

        hidden_states = self.pre_layer_norm(hidden_states)

        query, key = self.in_proj_qk(hidden_states).chunk(2, dim=2)  # shape: [T, B, D]
        value = self.in_proj_v(hidden_states)  # shape: [T, B, D]

        query = query.reshape(query_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
        key = key.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
        value = value.view(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)

        attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)

        pos = self.in_proj_qk(self.dropout(relative_embedding))  # shape: [2T-1, 2D]
        query_pos, key_pos = pos.view(-1, self.num_heads, 2*self.head_size).chunk(2, dim=2)
        query = query.view(batch_size, self.num_heads, query_len, self.head_size)
        key = key.view(batch_size, self.num_heads, query_len, self.head_size)

        attention_c_p = torch.einsum("bhqd,khd->bhqk", query, key_pos.squeeze(1) * self.scale)
        attention_p_c = torch.einsum("bhkd,qhd->bhqk", key * self.scale, query_pos.squeeze(1))

        position_indices = self.position_indices[:query_len, :key_len].expand(batch_size, self.num_heads, -1, -1)
        attention_c_p = attention_c_p.gather(3, position_indices)
        attention_p_c = attention_p_c.gather(2, position_indices)

        attention_scores = attention_scores.view(batch_size, self.num_heads, query_len, key_len)
        attention_scores.add_(attention_c_p)
        attention_scores.add_(attention_p_c)

        return attention_scores, value

    def compute_output(self, attention_probs, value):
        attention_probs = self.dropout(attention_probs)
        context = torch.bmm(attention_probs.flatten(0, 1), value)  # shape: [B*H, Q, D]
        context = context.transpose(0, 1).reshape(context.size(1), -1, self.hidden_size)  # shape: [Q, B, H*D]
        context = self.out_proj(context)
        context = self.post_layer_norm(context)
        context = self.dropout(context)
        return context

    def forward(self, hidden_states, attention_mask, relative_embedding):
        attention_scores, value = self.compute_attention_scores(hidden_states, relative_embedding)
        attention_probs = MaskedSoftmax.apply(attention_scores, attention_mask, -1)
        return self.compute_output(attention_probs, value), attention_probs.detach()


class Embedding(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.hidden_size = config.hidden_size

        self.word_embedding = nn.Embedding(config.vocab_size, config.hidden_size)
        self.word_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

        self.relative_embedding = nn.Parameter(torch.empty(2 * config.position_bucket_size - 1, config.hidden_size))
        self.relative_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)

        self.initialize()

    def initialize(self):
        std = math.sqrt(2.0 / (5.0 * self.hidden_size))
        nn.init.trunc_normal_(self.relative_embedding, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.word_embedding.weight, mean=0.0, std=std, a=-2*std, b=2*std)

    def forward(self, input_ids):
        word_embedding = self.dropout(self.word_layer_norm(self.word_embedding(input_ids)))
        relative_embeddings = self.relative_layer_norm(self.relative_embedding)
        return word_embedding, relative_embeddings


#
# HuggingFace wrappers
#

class NorbertPreTrainedModel(PreTrainedModel):
    config_class = NorbertConfig
    base_model_prefix = "norbert3"
    supports_gradient_checkpointing = True

    def _set_gradient_checkpointing(self, module, value=False):
        if isinstance(module, Encoder):
            module.activation_checkpointing = value

    def _init_weights(self, module):
        pass  # everything is already initialized


class NorbertModel(NorbertPreTrainedModel):
    def __init__(self, config, add_mlm_layer=False, gradient_checkpointing=False, **kwargs):
        super().__init__(config, **kwargs)
        self.config = config

        self.embedding = Embedding(config)
        self.transformer = Encoder(config, activation_checkpointing=gradient_checkpointing)
        self.classifier = MaskClassifier(config, self.embedding.word_embedding.weight) if add_mlm_layer else None

    def get_input_embeddings(self):
        return self.embedding.word_embedding

    def set_input_embeddings(self, value):
        self.embedding.word_embedding = value

    def get_contextualized_embeddings(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None
    ) -> List[torch.Tensor]:
        if input_ids is not None:
            input_shape = input_ids.size()
        else:
            raise ValueError("You have to specify input_ids")

        batch_size, seq_length = input_shape
        device = input_ids.device

        if attention_mask is None:
            attention_mask = torch.zeros(batch_size, seq_length, dtype=torch.bool, device=device)
        else:
            attention_mask = ~attention_mask.bool()
        attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
 
        static_embeddings, relative_embedding = self.embedding(input_ids.t())
        contextualized_embeddings, attention_probs = self.transformer(static_embeddings, attention_mask, relative_embedding)
        contextualized_embeddings = [e.transpose(0, 1) for e in contextualized_embeddings]
        last_layer = contextualized_embeddings[-1]
        contextualized_embeddings = [contextualized_embeddings[0]] + [
            contextualized_embeddings[i] - contextualized_embeddings[i - 1]
            for i in range(1, len(contextualized_embeddings))
        ]
        return last_layer, contextualized_embeddings, attention_probs

    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        token_type_ids: Optional[torch.Tensor] = None,
        position_ids: Optional[torch.Tensor] = None,
        output_hidden_states: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        **kwargs
    ) -> Union[Tuple[torch.Tensor], BaseModelOutput]:
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)

        if not return_dict:
            return (
                sequence_output,
                *([contextualized_embeddings] if output_hidden_states else []),
                *([attention_probs] if output_attentions else [])
            )

        return BaseModelOutput(
            last_hidden_state=sequence_output,
            hidden_states=contextualized_embeddings if output_hidden_states else None,
            attentions=attention_probs if output_attentions else None
        )


class Classifier(nn.Module):
    def __init__(self, hidden_size, vocab_size, dropout):
        super().__init__()

        self.transform = nn.Sequential(
            nn.Linear(hidden_size, hidden_size),
            nn.GELU(),
            nn.LayerNorm(hidden_size, elementwise_affine=False),
            nn.Dropout(dropout),
            nn.Linear(hidden_size, vocab_size)
        )
        self.initialize(hidden_size)

    def initialize(self, hidden_size):
        std = math.sqrt(2.0 / (5.0 * hidden_size))
        nn.init.trunc_normal_(self.transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.transform[-1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
        self.transform[0].bias.data.zero_()
        self.transform[-1].bias.data.zero_()

    def forward(self, x):
        return self.transform(x)


class ZeroClassifier(nn.Module):
    def forward(self, x):
        output = torch.zeros(x.size(0), x.size(1), 2, device=x.device, dtype=x.dtype)
        output[:, :, 0] = 1.0
        output[:, :, 1] = -1.0
        return output


class EdgeClassifier(nn.Module):
    def __init__(self, hidden_size, dep_hidden_size, vocab_size, dropout):
        super().__init__()

        self.head_dep_transform = nn.Sequential(
            nn.Linear(hidden_size, hidden_size),
            nn.GELU(),
            nn.LayerNorm(hidden_size, elementwise_affine=False),
            nn.Dropout(dropout)
        )
        self.head_root_transform = nn.Sequential(
            nn.Linear(hidden_size, hidden_size),
            nn.GELU(),
            nn.LayerNorm(hidden_size, elementwise_affine=False),
            nn.Dropout(dropout)
        )
        self.head_bilinear = nn.Parameter(torch.zeros(hidden_size, hidden_size))
        self.head_linear_dep = nn.Linear(hidden_size, 1, bias=False)
        self.head_linear_root = nn.Linear(hidden_size, 1, bias=False)
        self.head_bias = nn.Parameter(torch.zeros(1))

        self.dep_dep_transform = nn.Sequential(
            nn.Linear(hidden_size, dep_hidden_size),
            nn.GELU(),
            nn.LayerNorm(dep_hidden_size, elementwise_affine=False),
            nn.Dropout(dropout)
        )
        self.dep_root_transform = nn.Sequential(
            nn.Linear(hidden_size, dep_hidden_size),
            nn.GELU(),
            nn.LayerNorm(dep_hidden_size, elementwise_affine=False),
            nn.Dropout(dropout)
        )
        self.dep_bilinear = nn.Parameter(torch.zeros(dep_hidden_size, dep_hidden_size, vocab_size))
        self.dep_linear_dep = nn.Linear(dep_hidden_size, vocab_size, bias=False)
        self.dep_linear_root = nn.Linear(dep_hidden_size, vocab_size, bias=False)
        self.dep_bias = nn.Parameter(torch.zeros(vocab_size))

        self.hidden_size = hidden_size
        self.dep_hidden_size = dep_hidden_size

        self.mask_value = float("-inf")
        self.initialize(hidden_size)

    def initialize(self, hidden_size):
        std = math.sqrt(2.0 / (5.0 * hidden_size))
        nn.init.trunc_normal_(self.head_dep_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.head_root_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.dep_dep_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.dep_root_transform[0].weight, mean=0.0, std=std, a=-2*std, b=2*std)

        nn.init.trunc_normal_(self.head_linear_dep.weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.head_linear_root.weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.dep_linear_dep.weight, mean=0.0, std=std, a=-2*std, b=2*std)
        nn.init.trunc_normal_(self.dep_linear_root.weight, mean=0.0, std=std, a=-2*std, b=2*std)

        self.head_dep_transform[0].bias.data.zero_()
        self.head_root_transform[0].bias.data.zero_()
        self.dep_dep_transform[0].bias.data.zero_()
        self.dep_root_transform[0].bias.data.zero_()

    def forward(self, head_x, dep_x, lengths, head_gold=None):
        head_dep = self.head_dep_transform(head_x[:, 1:, :])
        head_root = self.head_root_transform(head_x)
        head_prediction = torch.einsum("bkn,nm,blm->bkl", head_dep, self.head_bilinear, head_root / math.sqrt(self.hidden_size)) \
            + self.head_linear_dep(head_dep) + self.head_linear_root(head_root).transpose(1, 2) + self.head_bias

        mask = (torch.arange(head_x.size(1)).unsqueeze(0) >= lengths.unsqueeze(1)).unsqueeze(1).to(head_x.device)
        mask = mask | (torch.ones(head_x.size(1) - 1, head_x.size(1), dtype=torch.bool, device=head_x.device).tril(1) & torch.ones(head_x.size(1) - 1, head_x.size(1), dtype=torch.bool, device=head_x.device).triu(1))
        head_prediction = head_prediction.masked_fill(mask, self.mask_value)

        if head_gold is None:
            head_logp = torch.log_softmax(head_prediction, dim=-1)
            head_logp = F.pad(head_logp, (0, 0, 1, 0), value=torch.nan).cpu()
            head_gold = []
            for i, length in enumerate(lengths.tolist()):
                head = self.max_spanning_tree(head_logp[i, :length, :length])
                head = head + ((head_x.size(1) - 1) - len(head)) * [0]
                head_gold.append(torch.tensor(head))
            head_gold = torch.stack(head_gold).to(head_x.device)

        dep_dep = self.dep_dep_transform(dep_x[:, 1:])
        dep_root = dep_x.gather(1, head_gold.unsqueeze(-1).expand(-1, -1, dep_x.size(-1)).clamp(min=0))
        dep_root = self.dep_root_transform(dep_root)
        dep_prediction = torch.einsum("btm,mnl,btn->btl", dep_dep, self.dep_bilinear, dep_root / math.sqrt(self.dep_hidden_size)) \
            + self.dep_linear_dep(dep_dep) + self.dep_linear_root(dep_root) + self.dep_bias

        return head_prediction, dep_prediction, head_gold
    
    def max_spanning_tree(self, weight_matrix):
        weight_matrix = weight_matrix.clone()
        # weight_matrix[:, 0] = torch.nan

        # we need to make sure that the root is the parent of a single node
        # first, we try to use the default weights, it should work in most cases
        parents, _ = dependency_decoding.chu_liu_edmonds(weight_matrix.numpy().astype(float))

        assert parents[0] == -1, f"{parents}\n{weight_matrix}"
        parents = parents[1:]

        # check if the root is the parent of a single node
        if parents.count(0) == 1:
            return parents
        
        # if not, we need to modify the weights and try all possibilities
        # we try to find the node that is the parent of the root
        best_score = float("-inf")
        best_parents = None

        for i in range(len(parents)):
            weight_matrix_mod = weight_matrix.clone()
            weight_matrix_mod[:i+1, 0] = torch.nan
            weight_matrix_mod[i+2:, 0] = torch.nan
            parents, score = dependency_decoding.chu_liu_edmonds(weight_matrix_mod.numpy().astype(float))
            parents = parents[1:]

            if score > best_score:
                best_score = score
                best_parents = parents

        def print_whole_matrix(matrix):
            for i in range(matrix.shape[0]):
                print(" ".join([str(x) for x in matrix[i]]))

        assert best_parents is not None, f"{best_parents}\n{print_whole_matrix(weight_matrix)}"
        return best_parents


class Model(nn.Module):
    def __init__(self, dataset):
        super().__init__()

#        config = BertConfig("../../configs/base.json")
#        self.bert = Bert(config)
#        checkpoint = torch.load("../../checkpoints/test_wd=0.01/model.bin", map_location="cpu")
#        self.bert.load_state_dict(checkpoint["model"], strict=False)

        config = NorbertConfig.from_json_file("config.json")
        self.bert = NorbertModel(config)

        self.n_layers = config.num_hidden_layers

        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.layer_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False)
        self.upos_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
        self.xpos_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
        self.feats_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
        self.lemma_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
        self.head_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
        self.dep_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))
        self.ner_layer_score = nn.Parameter(torch.zeros(self.n_layers + 1, dtype=torch.float))

        self.lemma_classifier = Classifier(config.hidden_size, len(dataset.lemma_vocab), config.hidden_dropout_prob)
        self.upos_classifier = Classifier(config.hidden_size, len(dataset.upos_vocab), config.hidden_dropout_prob) if len(dataset.upos_vocab) > 2 else ZeroClassifier()
        self.xpos_classifier = Classifier(config.hidden_size, len(dataset.xpos_vocab), config.hidden_dropout_prob) if len(dataset.xpos_vocab) > 2 else ZeroClassifier()
        self.feats_classifier = Classifier(config.hidden_size, len(dataset.feats_vocab), config.hidden_dropout_prob) if len(dataset.feats_vocab) > 2 else ZeroClassifier()
        self.edge_classifier = EdgeClassifier(config.hidden_size, 128, len(dataset.arc_dep_vocab), config.hidden_dropout_prob)
        self.ner_classifier = Classifier(config.hidden_size, len(dataset.ne_vocab), config.hidden_dropout_prob) if len(dataset.ne_vocab) > 2 else ZeroClassifier()

    def forward(self, x, alignment_mask, subword_lengths, word_lengths, head_gold=None):
        padding_mask = (torch.arange(x.size(1)).unsqueeze(0) < subword_lengths.unsqueeze(1)).to(x.device)
        x = self.bert(x, padding_mask, output_hidden_states=True).hidden_states
        x = torch.stack(x, dim=0)

        upos_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.upos_layer_score, dim=0))
        xpos_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.xpos_layer_score, dim=0))
        feats_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.feats_layer_score, dim=0))
        lemma_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.lemma_layer_score, dim=0))
        head_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.head_layer_score, dim=0))
        dep_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.dep_layer_score, dim=0))
        ne_x = torch.einsum("lbtd, l -> btd", x, torch.softmax(self.ner_layer_score, dim=0))

        upos_x = torch.einsum("bsd,bst->btd", upos_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
        xpos_x = torch.einsum("bsd,bst->btd", xpos_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
        feats_x = torch.einsum("bsd,bst->btd", feats_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
        lemma_x = torch.einsum("bsd,bst->btd", lemma_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
        head_x = torch.einsum("bsd,bst->btd", head_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
        dep_x = torch.einsum("bsd,bst->btd", dep_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)
        ne_x = torch.einsum("bsd, bst -> btd", ne_x, alignment_mask) / alignment_mask.sum(1).unsqueeze(-1).clamp(min=1.0)

        upos_x = self.dropout(self.layer_norm(upos_x[:, 1:-1, :]))
        xpos_x = self.dropout(self.layer_norm(xpos_x[:, 1:-1, :]))
        feats_x = self.dropout(self.layer_norm(feats_x[:, 1:-1, :]))
        lemma_x = self.dropout(self.layer_norm(lemma_x[:, 1:-1, :]))
        head_x = self.dropout(self.layer_norm(head_x[:, 0:-1, :]))
        dep_x = self.dropout(self.layer_norm(dep_x[:, 0:-1, :]))
        ne_x = self.dropout(self.layer_norm(ne_x[:, 1:-1, :]))

        lemma_preds = self.lemma_classifier(lemma_x)
        upos_preds = self.upos_classifier(upos_x)
        xpos_preds = self.xpos_classifier(xpos_x)
        feats_preds = self.feats_classifier(feats_x)
        ne_preds = self.ner_classifier(feats_x)
        head_prediction, dep_prediction, head_liu = self.edge_classifier(head_x, dep_x, word_lengths, head_gold)

        return lemma_preds, upos_preds, xpos_preds, feats_preds, head_prediction, dep_prediction, ne_preds, head_liu


class Parser:
    def __init__(self):
        checkpoint = torch.load("checkpoint.bin", map_location="cpu")

        self.dataset = Dataset()
        self.dataset.load_state_dict(checkpoint["dataset"])

        self.model = Model(self.dataset)
        self.model.load_state_dict(checkpoint["model"])
        self.model.eval()
        del checkpoint

        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.model.to(self.device)

    def parse(self, sentence):
        sentence = ftfy.fix_text(sentence.strip())
        forms, subwords, alignment = self.dataset.prepare_input(sentence)

        with torch.no_grad():
            output = self.model(
                subwords.to(self.device),
                alignment.to(self.device),
                torch.tensor([len(forms) + 1], device=self.device),
                torch.tensor([subwords.size(1)], device=self.device)
            )
 
        lemma_p, upos_p, xpos_p, feats_p, _, dep_p, ne_p, head_p = output
        lemmas, upos, xpos, feats, heads, deprel, ne = self.dataset.decode_output(
            forms, lemma_p, upos_p, xpos_p, feats_p, dep_p, ne_p, head_p
        )

        return {
            "forms": forms,
            "lemmas": lemmas,
            "upos": upos,
            "xpos": xpos,
            "feats": feats,
            "heads": heads,
            "deprel": deprel,
            "ne": ne
        }