Upload 13 files

config.json

@@ -0,0 +1,43 @@
+{
+  "_name_or_path": "/home/bond005/competitions/SHROOM/models/xlm-roberta-xl-hallucination-detector",
+  "architectures": [
+    "XLMRobertaXLForHierarchicalSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoConfig": "modeling_hierarchical_classifier.HierarchicalXLMRobertaXLConfig",
+    "AutoModelForSequenceClassification": "modeling_hierarchical_classifier.XLMRobertaXLForHierarchicalSequenceClassification",
+    "AutoModelForTextEncoding": "modeling_hierarchical_classifier.XLMRobertaXLForHierarchicalEmbedding"
+  },
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 2560,
+  "id2label": {
+    "0": "Not Hallucination",
+    "1": "Hallucination"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "label2id": {
+    "Hallucination": 1,
+    "Not Hallucination": 0
+  },
+  "label_smoothing": null,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "hierarchical-xlm-roberta-xl",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 36,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "temperature": 0.1,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250880
+}

handler.py

@@ -0,0 +1,31 @@
+import torch
+
+from typing import Any, Dict, Union
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # load model and tokenizer from path
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.model = AutoModelForSequenceClassification.from_pretrained(
+            path, device_map="auto", trust_remote_code=True
+        )
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, Union[str, float]]:
+        # process input
+        inputs = data.pop("inputs", data)
+
+        # preprocess
+        inputs = self.tokenizer(inputs, return_tensors="pt").to(self.device)
+
+        # pass inputs with all kwargs in data
+        logits = self.model(**inputs)[0]
+
+        # postprocess the prediction
+        predicted_class_id = int(torch.argmax(logits, dim=-1))
+        predicted_score = float(logits[0, predicted_class_id])
+        predicted_label = str(self.model.config.id2label[predicted_class_id])
+
+        return {'label': predicted_label, 'score': predicted_score}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e154c6173ccc9cdc2d208fe3f4fe7177342302feaee2dce36434eca703da8a35
+size 4958767392

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e88a4c1d5be403e7a7421b564383c9b9afada7240716110d8711f3a1deb90ea
+size 2006286972

model.safetensors.index.json

@@ -0,0 +1,593 @@
+{
+  "metadata": {
+    "total_size": 6964981836
+  },
+  "weight_map": {
+    "classifier.dense.bias": "model-00002-of-00002.safetensors",
+    "classifier.dense.weight": "model-00002-of-00002.safetensors",
+    "classifier.out_proj.bias": "model-00002-of-00002.safetensors",
+    "classifier.out_proj.weight": "model-00002-of-00002.safetensors",
+    "layer_weights.weight": "model-00002-of-00002.safetensors",
+    "roberta.embeddings.position_embeddings.weight": "model-00001-of-00002.safetensors",
+    "roberta.embeddings.token_type_embeddings.weight": "model-00001-of-00002.safetensors",
+    "roberta.embeddings.word_embeddings.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.0.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.0.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.1.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.10.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.11.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.12.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.13.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.14.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.15.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.16.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.17.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.18.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.19.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.2.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.20.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.21.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.22.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.23.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.23.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.23.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.23.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.23.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.24.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.25.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.26.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.27.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.28.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.29.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.3.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.3.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.30.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.30.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.31.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.32.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.33.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.34.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.LayerNorm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.LayerNorm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self.key.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self.key.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self.query.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self.query.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self.value.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self.value.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self_attn_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.attention.self_attn_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.intermediate.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.intermediate.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.output.dense.bias": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.35.output.dense.weight": "model-00002-of-00002.safetensors",
+    "roberta.encoder.layer.4.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.4.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.5.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.6.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.7.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.8.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.LayerNorm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.LayerNorm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.output.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self.key.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self.key.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self.query.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self.query.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self.value.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self.value.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self_attn_layer_norm.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.attention.self_attn_layer_norm.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.intermediate.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.intermediate.dense.weight": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.output.dense.bias": "model-00001-of-00002.safetensors",
+    "roberta.encoder.layer.9.output.dense.weight": "model-00001-of-00002.safetensors"
+  }
+}

modeling_hierarchical_classifier.py

@@ -0,0 +1,305 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+from torch.nn.modules.loss import _Loss
+from transformers import XLMRobertaXLPreTrainedModel, XLMRobertaXLModel, XLMRobertaXLConfig
+from transformers import AutoModelForSequenceClassification, AutoConfig
+from transformers.modeling_outputs import ModelOutput
+from pytorch_metric_learning.losses import NTXentLoss
+
+
+@dataclass
+class HierarchicalSequenceEmbedderOutput(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    embeddings: torch.FloatTensor = None
+    layer_embeddings: torch.FloatTensor = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+@dataclass
+class HierarchicalSequenceClassifierOutput(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    embeddings: torch.FloatTensor = None
+    layer_embeddings: torch.FloatTensor = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class HierarchicalXLMRobertaXLConfig(XLMRobertaXLConfig):
+    model_type = "hierarchical-xlm-roberta-xl"
+
+    def __init__(self, label_smoothing: Optional[float] = None, **kwargs):
+        super().__init__(**kwargs)
+        self.label_smoothing = label_smoothing
+
+
+class XLMRobertaXLHierarchicalClassificationHead(torch.nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = torch.nn.Linear(config.hidden_size, config.hidden_size)
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = torch.nn.Dropout(classifier_dropout)
+        self.out_proj = torch.nn.Linear(config.hidden_size, config.num_labels)
+
+    def forward(self, features, **kwargs):
+        x = self.dropout(features)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+        return x
+
+
+def distance_to_probability(distance: torch.Tensor, margin: float) -> torch.Tensor:
+    margin = torch.full(size=distance.size(), fill_value=margin,
+                        dtype=distance.dtype, device=distance.device, requires_grad=False)
+    p = (1.0 + torch.exp(-margin)) / (1.0 + torch.exp(distance - margin))
+    del margin
+    return p
+
+
+class DistanceBasedLogisticLoss(_Loss):
+    __constants__ = ['margin', 'reduction']
+    margin: float
+
+    def __init__(self, margin: float = 1.0, size_average=None, reduce=None, reduction: str = 'mean'):
+        super(DistanceBasedLogisticLoss, self).__init__(size_average, reduce, reduction)
+        self.margin = margin
+
+    def forward(self, inputs, targets):
+        inputs = inputs.view(-1)
+        targets = targets.to(inputs.dtype).view(-1)
+        p = distance_to_probability(inputs, self.margin)
+        return 1.0 - torch.nn.functional.binary_cross_entropy(input=p, target=targets, reduction=self.reduction)
+
+
+class LayerGatingNetwork(torch.nn.Module):
+    __constants__ = ['in_features']
+    in_features: int
+    weight: torch.Tensor
+
+    def __init__(self, in_features: int, device=None, dtype=None) -> None:
+        factory_kwargs = {'device': device, 'dtype': dtype}
+        super().__init__()
+        self.in_features = in_features
+        self.weight = torch.nn.Parameter(torch.empty((1, in_features), **factory_kwargs))
+        self.reset_parameters()
+
+    def reset_parameters(self) -> None:
+        initial_layer_weights = np.array(
+            [1.0 / (self.in_features - layer_idx) for layer_idx in range(self.in_features)],
+            dtype=np.float32
+        )
+        initial_layer_weights /= np.sum(initial_layer_weights)
+        initial_layer_weights_pt = torch.tensor(
+            initial_layer_weights.reshape((1, self.in_features)),
+            dtype=self.weight.dtype,
+            device=self.weight.device
+        )
+        del initial_layer_weights
+        self.weight = torch.nn.Parameter(initial_layer_weights_pt)
+        del initial_layer_weights_pt
+
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        return torch.nn.functional.linear(input, torch.softmax(self.weight, dim=-1))
+
+    def extra_repr(self) -> str:
+        return 'in_features={}'.format(self.in_features)
+
+
+class XLMRobertaXLForHierarchicalEmbedding(XLMRobertaXLPreTrainedModel, ABC):
+    config_class = HierarchicalXLMRobertaXLConfig
+
+    def __init__(self, config: HierarchicalXLMRobertaXLConfig):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.temperature = config.temperature
+        self.config = config
+
+        self.roberta = XLMRobertaXLModel(config, add_pooling_layer=False)
+        self.layer_weights = LayerGatingNetwork(in_features=config.num_hidden_layers)
+
+        self.init_weights()
+
+    def init_weights(self):
+        super().init_weights()
+        with torch.no_grad():
+            self.layer_weights.reset_parameters()
+
+    def forward(
+            self,
+            input_ids: Optional[torch.LongTensor] = None,
+            attention_mask: Optional[torch.FloatTensor] = None,
+            right_input_ids: Optional[torch.LongTensor] = None,
+            right_attention_mask: Optional[torch.LongTensor] = None,
+            token_type_ids: Optional[torch.LongTensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            head_mask: Optional[torch.FloatTensor] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            labels: Optional[torch.LongTensor] = None,
+            output_attentions: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, HierarchicalSequenceEmbedderOutput]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = self.roberta(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=True,
+            return_dict=False
+        )
+        cls_hidden_states = torch.stack(
+            tensors=outputs[2][-self.config.num_hidden_layers:],
+            dim=1
+        )[:, :, 0, :]
+        cls_emb = self.layer_weights(cls_hidden_states.permute(0, 2, 1))[:, :, 0]
+
+        loss = None
+        if labels is not None:
+            cls_emb_ = cls_emb.view(-1, self.config.hidden_size)
+            emb_norm = torch.linalg.norm(cls_emb_, dim=-1, keepdim=True) + 1e-9
+            if (right_input_ids is not None) or (right_attention_mask is not None):
+                if right_input_ids is None:
+                    raise ValueError(f'right_input_ids is not specified!')
+                if right_attention_mask is None:
+                    raise ValueError(f'right_attention_mask is not specified!')
+                right_outputs = self.roberta(
+                    right_input_ids,
+                    attention_mask=right_attention_mask,
+                    output_hidden_states=True,
+                    return_dict=False
+                )
+                right_cls_hidden_states = torch.stack(
+                    tensors=right_outputs[2][-self.config.num_hidden_layers:],
+                    dim=1
+                )[:, :, 0, :]
+                right_cls_emb = self.layer_weights(right_cls_hidden_states.permute(0, 2, 1))[:, :, 0]
+                right_cls_emb_ = right_cls_emb.view(-1, self.config.hidden_size)
+                right_emb_norm = torch.linalg.norm(right_cls_emb_, dim=-1, keepdim=True) + 1e-9
+                distances = torch.norm(cls_emb_ / emb_norm - right_cls_emb_ / right_emb_norm, 2, dim=-1)
+                loss_fct = DistanceBasedLogisticLoss(margin=1.0)
+                loss = loss_fct(distances, labels.view(-1))
+            else:
+                loss_fct = NTXentLoss(temperature=self.temperature)
+                loss = loss_fct(cls_emb_ / emb_norm, labels.view(-1))
+
+        if not return_dict:
+            output = (cls_emb, cls_hidden_states) + outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return HierarchicalSequenceEmbedderOutput(
+            loss=loss,
+            embeddings=cls_emb,
+            layer_embeddings=cls_hidden_states,
+            hidden_states=outputs[2],
+            attentions=outputs[3] if output_attentions else None,
+        )
+
+    @property
+    def layer_importances(self) -> List[Tuple[int, float]]:
+        with torch.no_grad():
+            importances = torch.softmax(self.layer_weights.weight, dim=-1).detach().cpu().numpy().flatten()
+        indices_and_importances = []
+        for layer_idx in range(importances.shape[0]):
+            indices_and_importances.append((layer_idx + 1, float(importances[layer_idx])))
+        indices_and_importances.sort(key=lambda it: (-it[1], it[0]))
+        return indices_and_importances
+
+
+class XLMRobertaXLForHierarchicalSequenceClassification(XLMRobertaXLForHierarchicalEmbedding, ABC):
+    def __init__(self, config: HierarchicalXLMRobertaXLConfig):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.label_smoothing = config.label_smoothing
+        self.config = config
+
+        self.classifier = XLMRobertaXLHierarchicalClassificationHead(config)
+
+        self.init_weights()
+
+    def forward(
+            self,
+            input_ids: Optional[torch.LongTensor] = None,
+            attention_mask: Optional[torch.FloatTensor] = None,
+            right_input_ids: Optional[torch.LongTensor] = None,
+            right_attention_mask: Optional[torch.LongTensor] = None,
+            token_type_ids: Optional[torch.LongTensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            head_mask: Optional[torch.FloatTensor] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            labels: Optional[torch.LongTensor] = None,
+            output_attentions: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, HierarchicalSequenceClassifierOutput]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = super().forward(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            return_dict=return_dict,
+        )
+        sequence_output = outputs[0]
+        logits = self.classifier(sequence_output)
+
+        loss = None
+        if labels is not None:
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = torch.nn.MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                if self.label_smoothing is None:
+                    loss_fct = torch.nn.CrossEntropyLoss()
+                else:
+                    loss_fct = torch.nn.CrossEntropyLoss(label_smoothing=self.label_smoothing)
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = torch.nn.BCEWithLogitsLoss()
+                loss = loss_fct(logits, labels)
+
+        if not return_dict:
+            output = (logits,) + outputs
+            return ((loss,) + output) if loss is not None else output
+
+        return HierarchicalSequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            embeddings=outputs.embeddings,
+            layer_embeddings=outputs.layer_embeddings,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions
+        )
+
+
+AutoConfig.register("hierarchical-xlm-roberta-xl", HierarchicalXLMRobertaXLConfig)
+AutoModelForSequenceClassification.register(
+    HierarchicalXLMRobertaXLConfig,
+    XLMRobertaXLForHierarchicalSequenceClassification
+)

pytorch_model-00001-of-00003.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:468d5860d463d60e770c77dc4c93593da6dd815a6a5a4e547935ebc367d7b526
+size 4987088064

pytorch_model-00002-of-00003.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d195a985039c4940ec45052ba1320a200af3179a964668ae14301a225312b1f
+size 4930484493

pytorch_model-00003-of-00003.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cda0ce53bfc8397e4d3872733b30450fc62734860f2b668606f40a49f2ad9e2
+size 4012594338

pytorch_model.bin.index.json

@@ -0,0 +1,593 @@
+{
+  "metadata": {
+    "total_size": 13929963672
+  },
+  "weight_map": {
+    "classifier.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "classifier.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "classifier.out_proj.bias": "pytorch_model-00003-of-00003.bin",
+    "classifier.out_proj.weight": "pytorch_model-00003-of-00003.bin",
+    "layer_weights.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.embeddings.position_embeddings.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.embeddings.token_type_embeddings.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.embeddings.word_embeddings.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.0.LayerNorm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.LayerNorm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.0.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.LayerNorm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.LayerNorm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.1.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.10.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.10.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.11.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.12.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.13.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.14.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.15.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.16.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.17.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.18.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.19.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.2.LayerNorm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.LayerNorm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.2.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.20.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.20.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.21.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.22.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.23.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.23.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.23.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.23.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.23.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.23.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.24.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.25.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.26.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.27.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.28.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.29.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.3.LayerNorm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.LayerNorm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.3.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.30.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.30.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.31.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.32.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.33.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.34.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.LayerNorm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.LayerNorm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self.key.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self.key.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self.query.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self.query.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self.value.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self.value.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self_attn_layer_norm.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.attention.self_attn_layer_norm.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.intermediate.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.intermediate.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.output.dense.bias": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.35.output.dense.weight": "pytorch_model-00003-of-00003.bin",
+    "roberta.encoder.layer.4.LayerNorm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.LayerNorm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.4.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.LayerNorm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.LayerNorm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.5.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.LayerNorm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.LayerNorm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.6.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.7.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.7.attention.output.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.output.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self.key.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self.key.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self.query.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self.query.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self.value.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self.value.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self_attn_layer_norm.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.attention.self_attn_layer_norm.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.intermediate.dense.bias": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.intermediate.dense.weight": "pytorch_model-00001-of-00003.bin",
+    "roberta.encoder.layer.7.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.7.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.8.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.LayerNorm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.LayerNorm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.output.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self.key.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self.key.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self.query.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self.query.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self.value.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self.value.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self_attn_layer_norm.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.attention.self_attn_layer_norm.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.intermediate.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.intermediate.dense.weight": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.output.dense.bias": "pytorch_model-00002-of-00003.bin",
+    "roberta.encoder.layer.9.output.dense.weight": "pytorch_model-00002-of-00003.bin"
+  }
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}