Omar commited on
Commit
ecd6e41
1 Parent(s): 3b8ebdc
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +4 -2
  2. finetune/boolq/all_results.json +16 -0
  3. finetune/boolq/config.json +50 -0
  4. finetune/boolq/eval_results.json +11 -0
  5. finetune/boolq/merges.txt +0 -0
  6. finetune/boolq/modeling_structroberta.py +2146 -0
  7. finetune/boolq/predict_results.txt +724 -0
  8. finetune/boolq/pytorch_model.bin +3 -0
  9. finetune/boolq/special_tokens_map.json +15 -0
  10. finetune/boolq/tokenizer_config.json +65 -0
  11. finetune/boolq/train_results.json +8 -0
  12. finetune/boolq/trainer_state.json +25 -0
  13. finetune/boolq/training_args.bin +3 -0
  14. finetune/boolq/vocab.json +0 -0
  15. finetune/cola/all_results.json +16 -0
  16. finetune/cola/config.json +50 -0
  17. finetune/cola/eval_results.json +11 -0
  18. finetune/cola/merges.txt +0 -0
  19. finetune/cola/modeling_structroberta.py +2146 -0
  20. finetune/cola/predict_results.txt +1020 -0
  21. finetune/cola/pytorch_model.bin +3 -0
  22. finetune/cola/special_tokens_map.json +15 -0
  23. finetune/cola/tokenizer_config.json +65 -0
  24. finetune/cola/train_results.json +8 -0
  25. finetune/cola/trainer_state.json +42 -0
  26. finetune/cola/training_args.bin +3 -0
  27. finetune/cola/vocab.json +0 -0
  28. finetune/control_raising_control/all_results.json +16 -0
  29. finetune/control_raising_control/config.json +50 -0
  30. finetune/control_raising_control/eval_results.json +11 -0
  31. finetune/control_raising_control/merges.txt +0 -0
  32. finetune/control_raising_control/modeling_structroberta.py +2146 -0
  33. finetune/control_raising_control/predict_results.txt +0 -0
  34. finetune/control_raising_control/pytorch_model.bin +3 -0
  35. finetune/control_raising_control/special_tokens_map.json +15 -0
  36. finetune/control_raising_control/tokenizer_config.json +65 -0
  37. finetune/control_raising_control/train_results.json +8 -0
  38. finetune/control_raising_control/trainer_state.json +42 -0
  39. finetune/control_raising_control/training_args.bin +3 -0
  40. finetune/control_raising_control/vocab.json +0 -0
  41. finetune/control_raising_lexical_content_the/all_results.json +16 -0
  42. finetune/control_raising_lexical_content_the/config.json +50 -0
  43. finetune/control_raising_lexical_content_the/eval_results.json +11 -0
  44. finetune/control_raising_lexical_content_the/merges.txt +0 -0
  45. finetune/control_raising_lexical_content_the/modeling_structroberta.py +2146 -0
  46. finetune/control_raising_lexical_content_the/predict_results.txt +0 -0
  47. finetune/control_raising_lexical_content_the/pytorch_model.bin +3 -0
  48. finetune/control_raising_lexical_content_the/special_tokens_map.json +15 -0
  49. finetune/control_raising_lexical_content_the/tokenizer_config.json +65 -0
  50. finetune/control_raising_lexical_content_the/train_results.json +8 -0
config.json CHANGED
@@ -1,11 +1,13 @@
1
  {
2
  "architectures": [
3
- "StructRoberta"
 
4
  ],
5
  "attention_probs_dropout_prob": 0.1,
6
  "auto_map": {
7
  "AutoConfig": "modeling_structroberta.StructRobertaConfig",
8
- "AutoModelForMaskedLM": "modeling_structroberta.StructRoberta"
 
9
  },
10
  "bos_token_id": 0,
11
  "classifier_dropout": null,
 
1
  {
2
  "architectures": [
3
+ "StructRoberta",
4
+ "StructRobertaForSequenceClassification"
5
  ],
6
  "attention_probs_dropout_prob": 0.1,
7
  "auto_map": {
8
  "AutoConfig": "modeling_structroberta.StructRobertaConfig",
9
+ "AutoModelForMaskedLM": "modeling_structroberta.StructRoberta",
10
+ "AutoModelForSequenceClassification": "modeling_structroberta.StructRobertaForSequenceClassification"
11
  },
12
  "bos_token_id": 0,
13
  "classifier_dropout": null,
finetune/boolq/all_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.658367931842804,
4
+ "eval_f1": 0.7413612565445026,
5
+ "eval_loss": 1.2280696630477905,
6
+ "eval_mcc": 0.2606317362733268,
7
+ "eval_runtime": 1.4424,
8
+ "eval_samples": 723,
9
+ "eval_samples_per_second": 501.248,
10
+ "eval_steps_per_second": 63.089,
11
+ "train_loss": 0.311509641011556,
12
+ "train_runtime": 90.5546,
13
+ "train_samples": 2072,
14
+ "train_samples_per_second": 228.812,
15
+ "train_steps_per_second": 1.988
16
+ }
finetune/boolq/config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "final_models/glue_models/structroberta_s2_final/",
3
+ "architectures": [
4
+ "StructRobertaForSequenceClassification"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "auto_map": {
8
+ "AutoConfig": "modeling_structroberta.StructRobertaConfig",
9
+ "AutoModelForMaskedLM": "modeling_structroberta.StructRoberta",
10
+ "AutoModelForSequenceClassification": "modeling_structroberta.StructRobertaForSequenceClassification"
11
+ },
12
+ "bos_token_id": 0,
13
+ "classifier_dropout": null,
14
+ "conv_size": 9,
15
+ "eos_token_id": 2,
16
+ "hidden_act": "gelu",
17
+ "hidden_dropout_prob": 0.1,
18
+ "hidden_size": 768,
19
+ "id2label": {
20
+ "0": 0,
21
+ "1": 1
22
+ },
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 3072,
25
+ "label2id": {
26
+ "0": 0,
27
+ "1": 1
28
+ },
29
+ "layer_norm_eps": 1e-05,
30
+ "max_position_embeddings": 514,
31
+ "model_type": "roberta",
32
+ "n_cntxt_layers": 4,
33
+ "n_cntxt_layers_2": 0,
34
+ "n_parser_layers": 4,
35
+ "num_attention_heads": 12,
36
+ "num_hidden_layers": 8,
37
+ "pad_token_id": 1,
38
+ "position_embedding_type": "absolute",
39
+ "problem_type": "single_label_classification",
40
+ "relations": [
41
+ "head",
42
+ "child"
43
+ ],
44
+ "torch_dtype": "float32",
45
+ "transformers_version": "4.26.1",
46
+ "type_vocab_size": 1,
47
+ "use_cache": true,
48
+ "vocab_size": 32000,
49
+ "weight_act": "softmax"
50
+ }
finetune/boolq/eval_results.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.658367931842804,
4
+ "eval_f1": 0.7413612565445026,
5
+ "eval_loss": 1.2280696630477905,
6
+ "eval_mcc": 0.2606317362733268,
7
+ "eval_runtime": 1.4424,
8
+ "eval_samples": 723,
9
+ "eval_samples_per_second": 501.248,
10
+ "eval_steps_per_second": 63.089
11
+ }
finetune/boolq/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
finetune/boolq/modeling_structroberta.py ADDED
@@ -0,0 +1,2146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import math
3
+ from typing import List, Optional, Tuple, Union
4
+ import torch
5
+ import torch.utils.checkpoint
6
+ from packaging import version
7
+ from torch import nn
8
+ import torch.nn.functional as F
9
+ from torch.nn import CrossEntropyLoss, MSELoss, BCEWithLogitsLoss
10
+ from transformers.activations import ACT2FN, gelu
11
+ from transformers.modeling_outputs import (
12
+ BaseModelOutputWithPastAndCrossAttentions,
13
+ BaseModelOutputWithPoolingAndCrossAttentions,
14
+ MaskedLMOutput,
15
+ SequenceClassifierOutput
16
+ )
17
+ from transformers.modeling_utils import (
18
+ PreTrainedModel,
19
+ apply_chunking_to_forward,
20
+ find_pruneable_heads_and_indices,
21
+ prune_linear_layer,
22
+ )
23
+ from transformers.utils import logging
24
+ from transformers import RobertaConfig
25
+
26
+
27
+ logger = logging.get_logger(__name__)
28
+ ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [
29
+ "roberta-base",
30
+ "roberta-large",
31
+ "roberta-large-mnli",
32
+ "distilroberta-base",
33
+ "roberta-base-openai-detector",
34
+ "roberta-large-openai-detector",
35
+ # See all RoBERTa models at https://huggingface.co/models?filter=roberta
36
+ ]
37
+
38
+
39
+ class StructRobertaConfig(RobertaConfig):
40
+ model_type = "roberta"
41
+
42
+ def __init__(
43
+ self,
44
+ n_parser_layers=4,
45
+ conv_size=9,
46
+ relations=("head", "child"),
47
+ weight_act="softmax",
48
+ n_cntxt_layers=3,
49
+ n_cntxt_layers_2=0,
50
+ **kwargs,):
51
+
52
+ super().__init__(**kwargs)
53
+ self.n_cntxt_layers = n_cntxt_layers
54
+ self.n_parser_layers = n_parser_layers
55
+ self.n_cntxt_layers_2 = n_cntxt_layers_2
56
+ self.conv_size = conv_size
57
+ self.relations = relations
58
+ self.weight_act = weight_act
59
+
60
+ class Conv1d(nn.Module):
61
+ """1D convolution layer."""
62
+
63
+ def __init__(self, hidden_size, kernel_size, dilation=1):
64
+ """Initialization.
65
+
66
+ Args:
67
+ hidden_size: dimension of input embeddings
68
+ kernel_size: convolution kernel size
69
+ dilation: the spacing between the kernel points
70
+ """
71
+ super(Conv1d, self).__init__()
72
+
73
+ if kernel_size % 2 == 0:
74
+ padding = (kernel_size // 2) * dilation
75
+ self.shift = True
76
+ else:
77
+ padding = ((kernel_size - 1) // 2) * dilation
78
+ self.shift = False
79
+ self.conv = nn.Conv1d(
80
+ hidden_size, hidden_size, kernel_size, padding=padding, dilation=dilation
81
+ )
82
+
83
+ def forward(self, x):
84
+ """Compute convolution.
85
+
86
+ Args:
87
+ x: input embeddings
88
+ Returns:
89
+ conv_output: convolution results
90
+ """
91
+
92
+ if self.shift:
93
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)[:, 1:]
94
+ else:
95
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)
96
+
97
+
98
+ class RobertaEmbeddings(nn.Module):
99
+ """
100
+ Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
101
+ """
102
+
103
+ # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__
104
+ def __init__(self, config):
105
+ super().__init__()
106
+ self.word_embeddings = nn.Embedding(
107
+ config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id
108
+ )
109
+ self.position_embeddings = nn.Embedding(
110
+ config.max_position_embeddings, config.hidden_size
111
+ )
112
+ self.token_type_embeddings = nn.Embedding(
113
+ config.type_vocab_size, config.hidden_size
114
+ )
115
+
116
+ # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
117
+ # any TensorFlow checkpoint file
118
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
119
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
120
+ # position_ids (1, len position emb) is contiguous in memory and exported when serialized
121
+ self.position_embedding_type = getattr(
122
+ config, "position_embedding_type", "absolute"
123
+ )
124
+ self.register_buffer(
125
+ "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))
126
+ )
127
+ if version.parse(torch.__version__) > version.parse("1.6.0"):
128
+ self.register_buffer(
129
+ "token_type_ids",
130
+ torch.zeros(self.position_ids.size(), dtype=torch.long),
131
+ persistent=False,
132
+ )
133
+
134
+ # End copy
135
+ self.padding_idx = config.pad_token_id
136
+ self.position_embeddings = nn.Embedding(
137
+ config.max_position_embeddings,
138
+ config.hidden_size,
139
+ padding_idx=self.padding_idx,
140
+ )
141
+
142
+ def forward(
143
+ self,
144
+ input_ids=None,
145
+ token_type_ids=None,
146
+ position_ids=None,
147
+ inputs_embeds=None,
148
+ past_key_values_length=0,
149
+ ):
150
+ if position_ids is None:
151
+ if input_ids is not None:
152
+ # Create the position ids from the input token ids. Any padded tokens remain padded.
153
+ position_ids = create_position_ids_from_input_ids(
154
+ input_ids, self.padding_idx, past_key_values_length
155
+ )
156
+ else:
157
+ position_ids = self.create_position_ids_from_inputs_embeds(
158
+ inputs_embeds
159
+ )
160
+
161
+ if input_ids is not None:
162
+ input_shape = input_ids.size()
163
+ else:
164
+ input_shape = inputs_embeds.size()[:-1]
165
+
166
+ seq_length = input_shape[1]
167
+
168
+ # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs
169
+ # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
170
+ # issue #5664
171
+ if token_type_ids is None:
172
+ if hasattr(self, "token_type_ids"):
173
+ buffered_token_type_ids = self.token_type_ids[:, :seq_length]
174
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
175
+ input_shape[0], seq_length
176
+ )
177
+ token_type_ids = buffered_token_type_ids_expanded
178
+ else:
179
+ token_type_ids = torch.zeros(
180
+ input_shape, dtype=torch.long, device=self.position_ids.device
181
+ )
182
+
183
+ if inputs_embeds is None:
184
+ inputs_embeds = self.word_embeddings(input_ids)
185
+ token_type_embeddings = self.token_type_embeddings(token_type_ids)
186
+
187
+ embeddings = inputs_embeds + token_type_embeddings
188
+ if self.position_embedding_type == "absolute":
189
+ position_embeddings = self.position_embeddings(position_ids)
190
+ embeddings += position_embeddings
191
+ embeddings = self.LayerNorm(embeddings)
192
+ embeddings = self.dropout(embeddings)
193
+ return embeddings
194
+
195
+ def create_position_ids_from_inputs_embeds(self, inputs_embeds):
196
+ """
197
+ We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.
198
+
199
+ Args:
200
+ inputs_embeds: torch.Tensor
201
+
202
+ Returns: torch.Tensor
203
+ """
204
+ input_shape = inputs_embeds.size()[:-1]
205
+ sequence_length = input_shape[1]
206
+
207
+ position_ids = torch.arange(
208
+ self.padding_idx + 1,
209
+ sequence_length + self.padding_idx + 1,
210
+ dtype=torch.long,
211
+ device=inputs_embeds.device,
212
+ )
213
+ return position_ids.unsqueeze(0).expand(input_shape)
214
+
215
+
216
+ # Copied from transformers.models.bert.modeling_bert.BertSelfAttention with Bert->Roberta
217
+ class RobertaSelfAttention(nn.Module):
218
+ def __init__(self, config, position_embedding_type=None):
219
+ super().__init__()
220
+ if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
221
+ config, "embedding_size"
222
+ ):
223
+ raise ValueError(
224
+ f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
225
+ f"heads ({config.num_attention_heads})"
226
+ )
227
+
228
+ self.num_attention_heads = config.num_attention_heads
229
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
230
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
231
+
232
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
233
+ self.key = nn.Linear(config.hidden_size, self.all_head_size)
234
+ self.value = nn.Linear(config.hidden_size, self.all_head_size)
235
+
236
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
237
+ self.position_embedding_type = position_embedding_type or getattr(
238
+ config, "position_embedding_type", "absolute"
239
+ )
240
+ if (
241
+ self.position_embedding_type == "relative_key"
242
+ or self.position_embedding_type == "relative_key_query"
243
+ ):
244
+ self.max_position_embeddings = config.max_position_embeddings
245
+ self.distance_embedding = nn.Embedding(
246
+ 2 * config.max_position_embeddings - 1, self.attention_head_size
247
+ )
248
+
249
+ self.is_decoder = config.is_decoder
250
+
251
+ def transpose_for_scores(self, x):
252
+ new_x_shape = x.size()[:-1] + (
253
+ self.num_attention_heads,
254
+ self.attention_head_size,
255
+ )
256
+ x = x.view(new_x_shape)
257
+ return x.permute(0, 2, 1, 3)
258
+
259
+ def forward(
260
+ self,
261
+ hidden_states: torch.Tensor,
262
+ attention_mask: Optional[torch.FloatTensor] = None,
263
+ head_mask: Optional[torch.FloatTensor] = None,
264
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
265
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
266
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
267
+ output_attentions: Optional[bool] = False,
268
+ parser_att_mask=None,
269
+ ) -> Tuple[torch.Tensor]:
270
+ mixed_query_layer = self.query(hidden_states)
271
+
272
+ # If this is instantiated as a cross-attention module, the keys
273
+ # and values come from an encoder; the attention mask needs to be
274
+ # such that the encoder's padding tokens are not attended to.
275
+ is_cross_attention = encoder_hidden_states is not None
276
+
277
+ if is_cross_attention and past_key_value is not None:
278
+ # reuse k,v, cross_attentions
279
+ key_layer = past_key_value[0]
280
+ value_layer = past_key_value[1]
281
+ attention_mask = encoder_attention_mask
282
+ elif is_cross_attention:
283
+ key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
284
+ value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
285
+ attention_mask = encoder_attention_mask
286
+ elif past_key_value is not None:
287
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
288
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
289
+ key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
290
+ value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
291
+ else:
292
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
293
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
294
+
295
+ query_layer = self.transpose_for_scores(mixed_query_layer)
296
+
297
+ if self.is_decoder:
298
+ # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
299
+ # Further calls to cross_attention layer can then reuse all cross-attention
300
+ # key/value_states (first "if" case)
301
+ # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
302
+ # all previous decoder key/value_states. Further calls to uni-directional self-attention
303
+ # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
304
+ # if encoder bi-directional self-attention `past_key_value` is always `None`
305
+ past_key_value = (key_layer, value_layer)
306
+
307
+ # Take the dot product between "query" and "key" to get the raw attention scores.
308
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
309
+
310
+ if (
311
+ self.position_embedding_type == "relative_key"
312
+ or self.position_embedding_type == "relative_key_query"
313
+ ):
314
+ seq_length = hidden_states.size()[1]
315
+ position_ids_l = torch.arange(
316
+ seq_length, dtype=torch.long, device=hidden_states.device
317
+ ).view(-1, 1)
318
+ position_ids_r = torch.arange(
319
+ seq_length, dtype=torch.long, device=hidden_states.device
320
+ ).view(1, -1)
321
+ distance = position_ids_l - position_ids_r
322
+ positional_embedding = self.distance_embedding(
323
+ distance + self.max_position_embeddings - 1
324
+ )
325
+ positional_embedding = positional_embedding.to(
326
+ dtype=query_layer.dtype
327
+ ) # fp16 compatibility
328
+
329
+ if self.position_embedding_type == "relative_key":
330
+ relative_position_scores = torch.einsum(
331
+ "bhld,lrd->bhlr", query_layer, positional_embedding
332
+ )
333
+ attention_scores = attention_scores + relative_position_scores
334
+ elif self.position_embedding_type == "relative_key_query":
335
+ relative_position_scores_query = torch.einsum(
336
+ "bhld,lrd->bhlr", query_layer, positional_embedding
337
+ )
338
+ relative_position_scores_key = torch.einsum(
339
+ "bhrd,lrd->bhlr", key_layer, positional_embedding
340
+ )
341
+ attention_scores = (
342
+ attention_scores
343
+ + relative_position_scores_query
344
+ + relative_position_scores_key
345
+ )
346
+
347
+ attention_scores = attention_scores / math.sqrt(self.attention_head_size)
348
+ if attention_mask is not None:
349
+ # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
350
+ attention_scores = attention_scores + attention_mask
351
+
352
+ if parser_att_mask is None:
353
+ # Normalize the attention scores to probabilities.
354
+ attention_probs = nn.functional.softmax(attention_scores, dim=-1)
355
+ else:
356
+ attention_probs = torch.sigmoid(attention_scores) * parser_att_mask
357
+
358
+ # This is actually dropping out entire tokens to attend to, which might
359
+ # seem a bit unusual, but is taken from the original Transformer paper.
360
+ attention_probs = self.dropout(attention_probs)
361
+
362
+ # Mask heads if we want to
363
+ if head_mask is not None:
364
+ attention_probs = attention_probs * head_mask
365
+
366
+ context_layer = torch.matmul(attention_probs, value_layer)
367
+
368
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
369
+ new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
370
+ context_layer = context_layer.view(new_context_layer_shape)
371
+
372
+ outputs = (
373
+ (context_layer, attention_probs) if output_attentions else (context_layer,)
374
+ )
375
+
376
+ if self.is_decoder:
377
+ outputs = outputs + (past_key_value,)
378
+ return outputs
379
+
380
+
381
+ # Copied from transformers.models.bert.modeling_bert.BertSelfOutput
382
+ class RobertaSelfOutput(nn.Module):
383
+ def __init__(self, config):
384
+ super().__init__()
385
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
386
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
387
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
388
+
389
+ def forward(
390
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
391
+ ) -> torch.Tensor:
392
+ hidden_states = self.dense(hidden_states)
393
+ hidden_states = self.dropout(hidden_states)
394
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
395
+ return hidden_states
396
+
397
+
398
+ # Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Roberta
399
+ class RobertaAttention(nn.Module):
400
+ def __init__(self, config, position_embedding_type=None):
401
+ super().__init__()
402
+ self.self = RobertaSelfAttention(
403
+ config, position_embedding_type=position_embedding_type
404
+ )
405
+ self.output = RobertaSelfOutput(config)
406
+ self.pruned_heads = set()
407
+
408
+ def prune_heads(self, heads):
409
+ if len(heads) == 0:
410
+ return
411
+ heads, index = find_pruneable_heads_and_indices(
412
+ heads,
413
+ self.self.num_attention_heads,
414
+ self.self.attention_head_size,
415
+ self.pruned_heads,
416
+ )
417
+
418
+ # Prune linear layers
419
+ self.self.query = prune_linear_layer(self.self.query, index)
420
+ self.self.key = prune_linear_layer(self.self.key, index)
421
+ self.self.value = prune_linear_layer(self.self.value, index)
422
+ self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
423
+
424
+ # Update hyper params and store pruned heads
425
+ self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
426
+ self.self.all_head_size = (
427
+ self.self.attention_head_size * self.self.num_attention_heads
428
+ )
429
+ self.pruned_heads = self.pruned_heads.union(heads)
430
+
431
+ def forward(
432
+ self,
433
+ hidden_states: torch.Tensor,
434
+ attention_mask: Optional[torch.FloatTensor] = None,
435
+ head_mask: Optional[torch.FloatTensor] = None,
436
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
437
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
438
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
439
+ output_attentions: Optional[bool] = False,
440
+ parser_att_mask=None,
441
+ ) -> Tuple[torch.Tensor]:
442
+ self_outputs = self.self(
443
+ hidden_states,
444
+ attention_mask,
445
+ head_mask,
446
+ encoder_hidden_states,
447
+ encoder_attention_mask,
448
+ past_key_value,
449
+ output_attentions,
450
+ parser_att_mask=parser_att_mask,
451
+ )
452
+ attention_output = self.output(self_outputs[0], hidden_states)
453
+ outputs = (attention_output,) + self_outputs[
454
+ 1:
455
+ ] # add attentions if we output them
456
+ return outputs
457
+
458
+
459
+ # Copied from transformers.models.bert.modeling_bert.BertIntermediate
460
+ class RobertaIntermediate(nn.Module):
461
+ def __init__(self, config):
462
+ super().__init__()
463
+ self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
464
+ if isinstance(config.hidden_act, str):
465
+ self.intermediate_act_fn = ACT2FN[config.hidden_act]
466
+ else:
467
+ self.intermediate_act_fn = config.hidden_act
468
+
469
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
470
+ hidden_states = self.dense(hidden_states)
471
+ hidden_states = self.intermediate_act_fn(hidden_states)
472
+ return hidden_states
473
+
474
+
475
+ # Copied from transformers.models.bert.modeling_bert.BertOutput
476
+ class RobertaOutput(nn.Module):
477
+ def __init__(self, config):
478
+ super().__init__()
479
+ self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
480
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
481
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
482
+
483
+ def forward(
484
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
485
+ ) -> torch.Tensor:
486
+ hidden_states = self.dense(hidden_states)
487
+ hidden_states = self.dropout(hidden_states)
488
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
489
+ return hidden_states
490
+
491
+
492
+ # Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Roberta
493
+ class RobertaLayer(nn.Module):
494
+ def __init__(self, config):
495
+ super().__init__()
496
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
497
+ self.seq_len_dim = 1
498
+ self.attention = RobertaAttention(config)
499
+ self.is_decoder = config.is_decoder
500
+ self.add_cross_attention = config.add_cross_attention
501
+ if self.add_cross_attention:
502
+ if not self.is_decoder:
503
+ raise ValueError(
504
+ f"{self} should be used as a decoder model if cross attention is added"
505
+ )
506
+ self.crossattention = RobertaAttention(
507
+ config, position_embedding_type="absolute"
508
+ )
509
+ self.intermediate = RobertaIntermediate(config)
510
+ self.output = RobertaOutput(config)
511
+
512
+ def forward(
513
+ self,
514
+ hidden_states: torch.Tensor,
515
+ attention_mask: Optional[torch.FloatTensor] = None,
516
+ head_mask: Optional[torch.FloatTensor] = None,
517
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
518
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
519
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
520
+ output_attentions: Optional[bool] = False,
521
+ parser_att_mask=None,
522
+ ) -> Tuple[torch.Tensor]:
523
+ # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
524
+ self_attn_past_key_value = (
525
+ past_key_value[:2] if past_key_value is not None else None
526
+ )
527
+ self_attention_outputs = self.attention(
528
+ hidden_states,
529
+ attention_mask,
530
+ head_mask,
531
+ output_attentions=output_attentions,
532
+ past_key_value=self_attn_past_key_value,
533
+ parser_att_mask=parser_att_mask,
534
+ )
535
+ attention_output = self_attention_outputs[0]
536
+
537
+ # if decoder, the last output is tuple of self-attn cache
538
+ if self.is_decoder:
539
+ outputs = self_attention_outputs[1:-1]
540
+ present_key_value = self_attention_outputs[-1]
541
+ else:
542
+ outputs = self_attention_outputs[
543
+ 1:
544
+ ] # add self attentions if we output attention weights
545
+
546
+ cross_attn_present_key_value = None
547
+ if self.is_decoder and encoder_hidden_states is not None:
548
+ if not hasattr(self, "crossattention"):
549
+ raise ValueError(
550
+ f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`"
551
+ )
552
+
553
+ # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
554
+ cross_attn_past_key_value = (
555
+ past_key_value[-2:] if past_key_value is not None else None
556
+ )
557
+ cross_attention_outputs = self.crossattention(
558
+ attention_output,
559
+ attention_mask,
560
+ head_mask,
561
+ encoder_hidden_states,
562
+ encoder_attention_mask,
563
+ cross_attn_past_key_value,
564
+ output_attentions,
565
+ )
566
+ attention_output = cross_attention_outputs[0]
567
+ outputs = (
568
+ outputs + cross_attention_outputs[1:-1]
569
+ ) # add cross attentions if we output attention weights
570
+
571
+ # add cross-attn cache to positions 3,4 of present_key_value tuple
572
+ cross_attn_present_key_value = cross_attention_outputs[-1]
573
+ present_key_value = present_key_value + cross_attn_present_key_value
574
+
575
+ layer_output = apply_chunking_to_forward(
576
+ self.feed_forward_chunk,
577
+ self.chunk_size_feed_forward,
578
+ self.seq_len_dim,
579
+ attention_output,
580
+ )
581
+ outputs = (layer_output,) + outputs
582
+
583
+ # if decoder, return the attn key/values as the last output
584
+ if self.is_decoder:
585
+ outputs = outputs + (present_key_value,)
586
+
587
+ return outputs
588
+
589
+ def feed_forward_chunk(self, attention_output):
590
+ intermediate_output = self.intermediate(attention_output)
591
+ layer_output = self.output(intermediate_output, attention_output)
592
+ return layer_output
593
+
594
+
595
+ # Copied from transformers.models.bert.modeling_bert.BertEncoder with Bert->Roberta
596
+ class RobertaEncoder(nn.Module):
597
+ def __init__(self, config):
598
+ super().__init__()
599
+ self.config = config
600
+ self.layer = nn.ModuleList(
601
+ [RobertaLayer(config) for _ in range(config.num_hidden_layers)]
602
+ )
603
+ self.gradient_checkpointing = False
604
+
605
+ def forward(
606
+ self,
607
+ hidden_states: torch.Tensor,
608
+ attention_mask: Optional[torch.FloatTensor] = None,
609
+ head_mask: Optional[torch.FloatTensor] = None,
610
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
611
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
612
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
613
+ use_cache: Optional[bool] = None,
614
+ output_attentions: Optional[bool] = False,
615
+ output_hidden_states: Optional[bool] = False,
616
+ return_dict: Optional[bool] = True,
617
+ parser_att_mask=None,
618
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]:
619
+ all_hidden_states = () if output_hidden_states else None
620
+ all_self_attentions = () if output_attentions else None
621
+ all_cross_attentions = (
622
+ () if output_attentions and self.config.add_cross_attention else None
623
+ )
624
+
625
+ next_decoder_cache = () if use_cache else None
626
+ for i, layer_module in enumerate(self.layer):
627
+ if output_hidden_states:
628
+ all_hidden_states = all_hidden_states + (hidden_states,)
629
+
630
+ layer_head_mask = head_mask[i] if head_mask is not None else None
631
+ past_key_value = past_key_values[i] if past_key_values is not None else None
632
+
633
+ if self.gradient_checkpointing and self.training:
634
+
635
+ if use_cache:
636
+ logger.warning(
637
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
638
+ )
639
+ use_cache = False
640
+
641
+ def create_custom_forward(module):
642
+ def custom_forward(*inputs):
643
+ return module(*inputs, past_key_value, output_attentions)
644
+
645
+ return custom_forward
646
+
647
+ layer_outputs = torch.utils.checkpoint.checkpoint(
648
+ create_custom_forward(layer_module),
649
+ hidden_states,
650
+ attention_mask,
651
+ layer_head_mask,
652
+ encoder_hidden_states,
653
+ encoder_attention_mask,
654
+ )
655
+ else:
656
+ if parser_att_mask is not None:
657
+ layer_outputs = layer_module(
658
+ hidden_states,
659
+ attention_mask,
660
+ layer_head_mask,
661
+ encoder_hidden_states,
662
+ encoder_attention_mask,
663
+ past_key_value,
664
+ output_attentions,
665
+ parser_att_mask=parser_att_mask[i])
666
+ else:
667
+ layer_outputs = layer_module(
668
+ hidden_states,
669
+ attention_mask,
670
+ layer_head_mask,
671
+ encoder_hidden_states,
672
+ encoder_attention_mask,
673
+ past_key_value,
674
+ output_attentions,
675
+ parser_att_mask=None)
676
+
677
+
678
+ hidden_states = layer_outputs[0]
679
+ if use_cache:
680
+ next_decoder_cache += (layer_outputs[-1],)
681
+ if output_attentions:
682
+ all_self_attentions = all_self_attentions + (layer_outputs[1],)
683
+ if self.config.add_cross_attention:
684
+ all_cross_attentions = all_cross_attentions + (layer_outputs[2],)
685
+
686
+ if output_hidden_states:
687
+ all_hidden_states = all_hidden_states + (hidden_states,)
688
+
689
+ if not return_dict:
690
+ return tuple(
691
+ v
692
+ for v in [
693
+ hidden_states,
694
+ next_decoder_cache,
695
+ all_hidden_states,
696
+ all_self_attentions,
697
+ all_cross_attentions,
698
+ ]
699
+ if v is not None
700
+ )
701
+ return BaseModelOutputWithPastAndCrossAttentions(
702
+ last_hidden_state=hidden_states,
703
+ past_key_values=next_decoder_cache,
704
+ hidden_states=all_hidden_states,
705
+ attentions=all_self_attentions,
706
+ cross_attentions=all_cross_attentions,
707
+ )
708
+
709
+
710
+ # Copied from transformers.models.bert.modeling_bert.BertPooler
711
+ class RobertaPooler(nn.Module):
712
+ def __init__(self, config):
713
+ super().__init__()
714
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
715
+ self.activation = nn.Tanh()
716
+
717
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
718
+ # We "pool" the model by simply taking the hidden state corresponding
719
+ # to the first token.
720
+ first_token_tensor = hidden_states[:, 0]
721
+ pooled_output = self.dense(first_token_tensor)
722
+ pooled_output = self.activation(pooled_output)
723
+ return pooled_output
724
+
725
+
726
+ class RobertaPreTrainedModel(PreTrainedModel):
727
+ """
728
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
729
+ models.
730
+ """
731
+
732
+ config_class = RobertaConfig
733
+ base_model_prefix = "roberta"
734
+ supports_gradient_checkpointing = True
735
+
736
+ # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights
737
+ def _init_weights(self, module):
738
+ """Initialize the weights"""
739
+ if isinstance(module, nn.Linear):
740
+ # Slightly different from the TF version which uses truncated_normal for initialization
741
+ # cf https://github.com/pytorch/pytorch/pull/5617
742
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
743
+ if module.bias is not None:
744
+ module.bias.data.zero_()
745
+ elif isinstance(module, nn.Embedding):
746
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
747
+ if module.padding_idx is not None:
748
+ module.weight.data[module.padding_idx].zero_()
749
+ elif isinstance(module, nn.LayerNorm):
750
+ if module.bias is not None:
751
+ module.bias.data.zero_()
752
+ module.weight.data.fill_(1.0)
753
+
754
+ def _set_gradient_checkpointing(self, module, value=False):
755
+ if isinstance(module, RobertaEncoder):
756
+ module.gradient_checkpointing = value
757
+
758
+ def update_keys_to_ignore(self, config, del_keys_to_ignore):
759
+ """Remove some keys from ignore list"""
760
+ if not config.tie_word_embeddings:
761
+ # must make a new list, or the class variable gets modified!
762
+ self._keys_to_ignore_on_save = [
763
+ k for k in self._keys_to_ignore_on_save if k not in del_keys_to_ignore
764
+ ]
765
+ self._keys_to_ignore_on_load_missing = [
766
+ k
767
+ for k in self._keys_to_ignore_on_load_missing
768
+ if k not in del_keys_to_ignore
769
+ ]
770
+
771
+
772
+ ROBERTA_START_DOCSTRING = r"""
773
+
774
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
775
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
776
+ etc.)
777
+
778
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
779
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
780
+ and behavior.
781
+
782
+ Parameters:
783
+ config ([`RobertaConfig`]): Model configuration class with all the parameters of the
784
+ model. Initializing with a config file does not load the weights associated with the model, only the
785
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
786
+ """
787
+
788
+
789
+ ROBERTA_INPUTS_DOCSTRING = r"""
790
+ Args:
791
+ input_ids (`torch.LongTensor` of shape `({0})`):
792
+ Indices of input sequence tokens in the vocabulary.
793
+
794
+ Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
795
+ [`PreTrainedTokenizer.__call__`] for details.
796
+
797
+ [What are input IDs?](../glossary#input-ids)
798
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
799
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
800
+
801
+ - 1 for tokens that are **not masked**,
802
+ - 0 for tokens that are **masked**.
803
+
804
+ [What are attention masks?](../glossary#attention-mask)
805
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
806
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
807
+ 1]`:
808
+
809
+ - 0 corresponds to a *sentence A* token,
810
+ - 1 corresponds to a *sentence B* token.
811
+
812
+ [What are token type IDs?](../glossary#token-type-ids)
813
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
814
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
815
+ config.max_position_embeddings - 1]`.
816
+
817
+ [What are position IDs?](../glossary#position-ids)
818
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
819
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
820
+
821
+ - 1 indicates the head is **not masked**,
822
+ - 0 indicates the head is **masked**.
823
+
824
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
825
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
826
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
827
+ model's internal embedding lookup matrix.
828
+ output_attentions (`bool`, *optional*):
829
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
830
+ tensors for more detail.
831
+ output_hidden_states (`bool`, *optional*):
832
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
833
+ more detail.
834
+ return_dict (`bool`, *optional*):
835
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
836
+ """
837
+
838
+
839
+ class RobertaModel(RobertaPreTrainedModel):
840
+ """
841
+
842
+ The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
843
+ cross-attention is added between the self-attention layers, following the architecture described in *Attention is
844
+ all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
845
+ Kaiser and Illia Polosukhin.
846
+
847
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
848
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
849
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
850
+
851
+ .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762
852
+
853
+ """
854
+
855
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
856
+
857
+ # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->Roberta
858
+ def __init__(self, config, add_pooling_layer=True):
859
+ super().__init__(config)
860
+ self.config = config
861
+
862
+ self.embeddings = RobertaEmbeddings(config)
863
+ self.encoder = RobertaEncoder(config)
864
+
865
+ self.pooler = RobertaPooler(config) if add_pooling_layer else None
866
+
867
+ # Initialize weights and apply final processing
868
+ self.post_init()
869
+
870
+ def get_input_embeddings(self):
871
+ return self.embeddings.word_embeddings
872
+
873
+ def set_input_embeddings(self, value):
874
+ self.embeddings.word_embeddings = value
875
+
876
+ def _prune_heads(self, heads_to_prune):
877
+ """
878
+ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
879
+ class PreTrainedModel
880
+ """
881
+ for layer, heads in heads_to_prune.items():
882
+ self.encoder.layer[layer].attention.prune_heads(heads)
883
+
884
+ # Copied from transformers.models.bert.modeling_bert.BertModel.forward
885
+ def forward(
886
+ self,
887
+ input_ids: Optional[torch.Tensor] = None,
888
+ attention_mask: Optional[torch.Tensor] = None,
889
+ token_type_ids: Optional[torch.Tensor] = None,
890
+ position_ids: Optional[torch.Tensor] = None,
891
+ head_mask: Optional[torch.Tensor] = None,
892
+ inputs_embeds: Optional[torch.Tensor] = None,
893
+ encoder_hidden_states: Optional[torch.Tensor] = None,
894
+ encoder_attention_mask: Optional[torch.Tensor] = None,
895
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
896
+ use_cache: Optional[bool] = None,
897
+ output_attentions: Optional[bool] = None,
898
+ output_hidden_states: Optional[bool] = None,
899
+ return_dict: Optional[bool] = None,
900
+ parser_att_mask=None,
901
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]:
902
+ r"""
903
+ encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
904
+ Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
905
+ the model is configured as a decoder.
906
+ encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
907
+ Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
908
+ the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:
909
+
910
+ - 1 for tokens that are **not masked**,
911
+ - 0 for tokens that are **masked**.
912
+ past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
913
+ Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
914
+
915
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
916
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
917
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
918
+ use_cache (`bool`, *optional*):
919
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
920
+ `past_key_values`).
921
+ """
922
+ output_attentions = (
923
+ output_attentions
924
+ if output_attentions is not None
925
+ else self.config.output_attentions
926
+ )
927
+ output_hidden_states = (
928
+ output_hidden_states
929
+ if output_hidden_states is not None
930
+ else self.config.output_hidden_states
931
+ )
932
+ return_dict = (
933
+ return_dict if return_dict is not None else self.config.use_return_dict
934
+ )
935
+
936
+ if self.config.is_decoder:
937
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
938
+ else:
939
+ use_cache = False
940
+
941
+ if input_ids is not None and inputs_embeds is not None:
942
+ raise ValueError(
943
+ "You cannot specify both input_ids and inputs_embeds at the same time"
944
+ )
945
+ elif input_ids is not None:
946
+ input_shape = input_ids.size()
947
+ elif inputs_embeds is not None:
948
+ input_shape = inputs_embeds.size()[:-1]
949
+ else:
950
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
951
+
952
+ batch_size, seq_length = input_shape
953
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
954
+
955
+ # past_key_values_length
956
+ past_key_values_length = (
957
+ past_key_values[0][0].shape[2] if past_key_values is not None else 0
958
+ )
959
+
960
+ if attention_mask is None:
961
+ attention_mask = torch.ones(
962
+ ((batch_size, seq_length + past_key_values_length)), device=device
963
+ )
964
+
965
+ if token_type_ids is None:
966
+ if hasattr(self.embeddings, "token_type_ids"):
967
+ buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
968
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
969
+ batch_size, seq_length
970
+ )
971
+ token_type_ids = buffered_token_type_ids_expanded
972
+ else:
973
+ token_type_ids = torch.zeros(
974
+ input_shape, dtype=torch.long, device=device
975
+ )
976
+
977
+ # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
978
+ # ourselves in which case we just need to make it broadcastable to all heads.
979
+ extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(
980
+ attention_mask, input_shape, device
981
+ )
982
+
983
+ # If a 2D or 3D attention mask is provided for the cross-attention
984
+ # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
985
+ if self.config.is_decoder and encoder_hidden_states is not None:
986
+ (
987
+ encoder_batch_size,
988
+ encoder_sequence_length,
989
+ _,
990
+ ) = encoder_hidden_states.size()
991
+ encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
992
+ if encoder_attention_mask is None:
993
+ encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
994
+ encoder_extended_attention_mask = self.invert_attention_mask(
995
+ encoder_attention_mask
996
+ )
997
+ else:
998
+ encoder_extended_attention_mask = None
999
+
1000
+ # Prepare head mask if needed
1001
+ # 1.0 in head_mask indicate we keep the head
1002
+ # attention_probs has shape bsz x n_heads x N x N
1003
+ # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
1004
+ # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
1005
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
1006
+
1007
+ embedding_output = self.embeddings(
1008
+ input_ids=input_ids,
1009
+ position_ids=position_ids,
1010
+ token_type_ids=token_type_ids,
1011
+ inputs_embeds=inputs_embeds,
1012
+ past_key_values_length=past_key_values_length,
1013
+ )
1014
+ encoder_outputs = self.encoder(
1015
+ embedding_output,
1016
+ attention_mask=extended_attention_mask,
1017
+ head_mask=head_mask,
1018
+ encoder_hidden_states=encoder_hidden_states,
1019
+ encoder_attention_mask=encoder_extended_attention_mask,
1020
+ past_key_values=past_key_values,
1021
+ use_cache=use_cache,
1022
+ output_attentions=output_attentions,
1023
+ output_hidden_states=output_hidden_states,
1024
+ return_dict=return_dict,
1025
+ parser_att_mask=parser_att_mask,
1026
+ )
1027
+ sequence_output = encoder_outputs[0]
1028
+ pooled_output = (
1029
+ self.pooler(sequence_output) if self.pooler is not None else None
1030
+ )
1031
+
1032
+ if not return_dict:
1033
+ return (sequence_output, pooled_output) + encoder_outputs[1:]
1034
+
1035
+ return BaseModelOutputWithPoolingAndCrossAttentions(
1036
+ last_hidden_state=sequence_output,
1037
+ pooler_output=pooled_output,
1038
+ past_key_values=encoder_outputs.past_key_values,
1039
+ hidden_states=encoder_outputs.hidden_states,
1040
+ attentions=encoder_outputs.attentions,
1041
+ cross_attentions=encoder_outputs.cross_attentions,
1042
+ )
1043
+
1044
+
1045
+ class StructRoberta(RobertaPreTrainedModel):
1046
+ _keys_to_ignore_on_save = [r"lm_head.decoder.weight", r"lm_head.decoder.bias"]
1047
+ _keys_to_ignore_on_load_missing = [
1048
+ r"position_ids",
1049
+ r"lm_head.decoder.weight",
1050
+ r"lm_head.decoder.bias",
1051
+ ]
1052
+ _keys_to_ignore_on_load_unexpected = [r"pooler"]
1053
+
1054
+ def __init__(self, config):
1055
+ super().__init__(config)
1056
+
1057
+ if config.is_decoder:
1058
+ logger.warning(
1059
+ "If you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for "
1060
+ "bi-directional self-attention."
1061
+ )
1062
+
1063
+
1064
+ if config.n_cntxt_layers > 0:
1065
+ config_cntxt = copy.deepcopy(config)
1066
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1067
+
1068
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1069
+
1070
+ if config.n_cntxt_layers_2 > 0:
1071
+ self.parser_layers_1 = nn.ModuleList(
1072
+ [
1073
+ nn.Sequential(
1074
+ Conv1d(config.hidden_size, config.conv_size),
1075
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1076
+ nn.Tanh(),
1077
+ )
1078
+ for i in range(int(config.n_parser_layers/2))
1079
+ ]
1080
+ )
1081
+
1082
+ self.distance_ff_1 = nn.Sequential(
1083
+ Conv1d(config.hidden_size, 2),
1084
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1085
+ nn.Tanh(),
1086
+ nn.Linear(config.hidden_size, 1),
1087
+ )
1088
+
1089
+ self.height_ff_1 = nn.Sequential(
1090
+ nn.Linear(config.hidden_size, config.hidden_size),
1091
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1092
+ nn.Tanh(),
1093
+ nn.Linear(config.hidden_size, 1),
1094
+ )
1095
+
1096
+ n_rel = len(config.relations)
1097
+ self._rel_weight_1 = nn.Parameter(
1098
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1099
+ )
1100
+ self._rel_weight_1.data.normal_(0, 0.1)
1101
+
1102
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1103
+
1104
+ config_cntxt_2 = copy.deepcopy(config)
1105
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1106
+
1107
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1108
+
1109
+
1110
+ self.parser_layers_2 = nn.ModuleList(
1111
+ [
1112
+ nn.Sequential(
1113
+ Conv1d(config.hidden_size, config.conv_size),
1114
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1115
+ nn.Tanh(),
1116
+ )
1117
+ for i in range(int(config.n_parser_layers/2))
1118
+ ]
1119
+ )
1120
+
1121
+ self.distance_ff_2 = nn.Sequential(
1122
+ Conv1d(config.hidden_size, 2),
1123
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1124
+ nn.Tanh(),
1125
+ nn.Linear(config.hidden_size, 1),
1126
+ )
1127
+
1128
+ self.height_ff_2 = nn.Sequential(
1129
+ nn.Linear(config.hidden_size, config.hidden_size),
1130
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1131
+ nn.Tanh(),
1132
+ nn.Linear(config.hidden_size, 1),
1133
+ )
1134
+
1135
+ n_rel = len(config.relations)
1136
+ self._rel_weight_2 = nn.Parameter(
1137
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1138
+ )
1139
+ self._rel_weight_2.data.normal_(0, 0.1)
1140
+
1141
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1142
+
1143
+ else:
1144
+ self.parser_layers = nn.ModuleList(
1145
+ [
1146
+ nn.Sequential(
1147
+ Conv1d(config.hidden_size, config.conv_size),
1148
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1149
+ nn.Tanh(),
1150
+ )
1151
+ for i in range(config.n_parser_layers)
1152
+ ]
1153
+ )
1154
+
1155
+ self.distance_ff = nn.Sequential(
1156
+ Conv1d(config.hidden_size, 2),
1157
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1158
+ nn.Tanh(),
1159
+ nn.Linear(config.hidden_size, 1),
1160
+ )
1161
+
1162
+ self.height_ff = nn.Sequential(
1163
+ nn.Linear(config.hidden_size, config.hidden_size),
1164
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1165
+ nn.Tanh(),
1166
+ nn.Linear(config.hidden_size, 1),
1167
+ )
1168
+
1169
+ n_rel = len(config.relations)
1170
+ self._rel_weight = nn.Parameter(
1171
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1172
+ )
1173
+ self._rel_weight.data.normal_(0, 0.1)
1174
+
1175
+ self._scaler = nn.Parameter(torch.zeros(2))
1176
+
1177
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1178
+
1179
+ if config.n_cntxt_layers > 0:
1180
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1181
+ if config.n_cntxt_layers_2 > 0:
1182
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1183
+
1184
+ self.lm_head = RobertaLMHead(config)
1185
+
1186
+ self.pad = config.pad_token_id
1187
+
1188
+ # The LM head weights require special treatment only when they are tied with the word embeddings
1189
+ self.update_keys_to_ignore(config, ["lm_head.decoder.weight"])
1190
+
1191
+ # Initialize weights and apply final processing
1192
+ self.post_init()
1193
+
1194
+ def get_output_embeddings(self):
1195
+ return self.lm_head.decoder
1196
+
1197
+ def set_output_embeddings(self, new_embeddings):
1198
+ self.lm_head.decoder = new_embeddings
1199
+
1200
+ @property
1201
+ def scaler(self):
1202
+ return self._scaler.exp()
1203
+
1204
+ @property
1205
+ def scaler_1(self):
1206
+ return self._scaler_1.exp()
1207
+
1208
+ @property
1209
+ def scaler_2(self):
1210
+ return self._scaler_2.exp()
1211
+
1212
+ @property
1213
+ def rel_weight(self):
1214
+ if self.config.weight_act == "sigmoid":
1215
+ return torch.sigmoid(self._rel_weight)
1216
+ elif self.config.weight_act == "softmax":
1217
+ return torch.softmax(self._rel_weight, dim=-1)
1218
+
1219
+ @property
1220
+ def rel_weight_1(self):
1221
+ if self.config.weight_act == "sigmoid":
1222
+ return torch.sigmoid(self._rel_weight_1)
1223
+ elif self.config.weight_act == "softmax":
1224
+ return torch.softmax(self._rel_weight_1, dim=-1)
1225
+
1226
+
1227
+ @property
1228
+ def rel_weight_2(self):
1229
+ if self.config.weight_act == "sigmoid":
1230
+ return torch.sigmoid(self._rel_weight_2)
1231
+ elif self.config.weight_act == "softmax":
1232
+ return torch.softmax(self._rel_weight_2, dim=-1)
1233
+
1234
+
1235
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1236
+ """Compute constituents from distance and height."""
1237
+
1238
+ if n_cntxt_layers>0:
1239
+ if n_cntxt_layers == 1:
1240
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1241
+ elif n_cntxt_layers == 2:
1242
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1243
+ else:
1244
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1245
+
1246
+ gamma = torch.sigmoid(-beta_logits)
1247
+ ones = torch.ones_like(gamma)
1248
+
1249
+ block_mask_left = cummin(
1250
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1251
+ )
1252
+ block_mask_left = block_mask_left - F.pad(
1253
+ block_mask_left[:, :, :-1], (1, 0), value=0
1254
+ )
1255
+ block_mask_left.tril_(0)
1256
+
1257
+ block_mask_right = cummin(
1258
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1259
+ )
1260
+ block_mask_right = block_mask_right - F.pad(
1261
+ block_mask_right[:, :, 1:], (0, 1), value=0
1262
+ )
1263
+ block_mask_right.triu_(0)
1264
+
1265
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1266
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1267
+ block_mask_right, reverse=True
1268
+ ).triu(1)
1269
+
1270
+ return block_p, block
1271
+
1272
+ def compute_head(self, height, n_cntxt_layers=0):
1273
+ """Estimate head for each constituent."""
1274
+
1275
+ _, length = height.size()
1276
+ if n_cntxt_layers>0:
1277
+ if n_cntxt_layers == 1:
1278
+ head_logits = height * self.scaler_1[1]
1279
+ elif n_cntxt_layers == 2:
1280
+ head_logits = height * self.scaler_2[1]
1281
+ else:
1282
+ head_logits = height * self.scaler[1]
1283
+ index = torch.arange(length, device=height.device)
1284
+
1285
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1286
+ index[None, None, :] <= index[None, :, None]
1287
+ )
1288
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1289
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1290
+
1291
+ head_p = torch.softmax(head_logits, dim=-1)
1292
+
1293
+ return head_p
1294
+
1295
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1296
+ """Parse input sentence.
1297
+
1298
+ Args:
1299
+ x: input tokens (required).
1300
+ pos: position for each token (optional).
1301
+ Returns:
1302
+ distance: syntactic distance
1303
+ height: syntactic height
1304
+ """
1305
+
1306
+ mask = x != self.pad
1307
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1308
+
1309
+ if embs is None:
1310
+ h = self.roberta.embeddings(x)
1311
+ else:
1312
+ h = embs
1313
+
1314
+ if n_cntxt_layers > 0:
1315
+ if n_cntxt_layers == 1:
1316
+ parser_layers = self.parser_layers_1
1317
+ height_ff = self.height_ff_1
1318
+ distance_ff = self.distance_ff_1
1319
+ elif n_cntxt_layers == 2:
1320
+ parser_layers = self.parser_layers_2
1321
+ height_ff = self.height_ff_2
1322
+ distance_ff = self.distance_ff_2
1323
+ for i in range(int(self.config.n_parser_layers/2)):
1324
+ h = h.masked_fill(~mask[:, :, None], 0)
1325
+ h = parser_layers[i](h)
1326
+
1327
+ height = height_ff(h).squeeze(-1)
1328
+ height.masked_fill_(~mask, -1e9)
1329
+
1330
+ distance = distance_ff(h).squeeze(-1)
1331
+ distance.masked_fill_(~mask_shifted, 1e9)
1332
+
1333
+ # Calbrating the distance and height to the same level
1334
+ length = distance.size(1)
1335
+ height_max = height[:, None, :].expand(-1, length, -1)
1336
+ height_max = torch.cummax(
1337
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1338
+ )[0].triu(0)
1339
+
1340
+ margin_left = torch.relu(
1341
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1342
+ )
1343
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1344
+ margin = torch.where(
1345
+ margin_left > margin_right, margin_right, margin_left
1346
+ ).triu(0)
1347
+
1348
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1349
+ margin.masked_fill_(~margin_mask, 0)
1350
+ margin = margin.max()
1351
+
1352
+ distance = distance - margin
1353
+ else:
1354
+ for i in range(self.config.n_parser_layers):
1355
+ h = h.masked_fill(~mask[:, :, None], 0)
1356
+ h = self.parser_layers[i](h)
1357
+
1358
+ height = self.height_ff(h).squeeze(-1)
1359
+ height.masked_fill_(~mask, -1e9)
1360
+
1361
+ distance = self.distance_ff(h).squeeze(-1)
1362
+ distance.masked_fill_(~mask_shifted, 1e9)
1363
+
1364
+ # Calbrating the distance and height to the same level
1365
+ length = distance.size(1)
1366
+ height_max = height[:, None, :].expand(-1, length, -1)
1367
+ height_max = torch.cummax(
1368
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1369
+ )[0].triu(0)
1370
+
1371
+ margin_left = torch.relu(
1372
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1373
+ )
1374
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1375
+ margin = torch.where(
1376
+ margin_left > margin_right, margin_right, margin_left
1377
+ ).triu(0)
1378
+
1379
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1380
+ margin.masked_fill_(~margin_mask, 0)
1381
+ margin = margin.max()
1382
+
1383
+ distance = distance - margin
1384
+
1385
+ return distance, height
1386
+
1387
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1388
+ """Compute head and cibling distribution for each token."""
1389
+
1390
+ bsz, length = x.size()
1391
+
1392
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1393
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1394
+
1395
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1396
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1397
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1398
+ head = head.masked_fill(eye, 0)
1399
+ child = head.transpose(1, 2)
1400
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1401
+
1402
+ rel_list = []
1403
+ if "head" in self.config.relations:
1404
+ rel_list.append(head)
1405
+ if "child" in self.config.relations:
1406
+ rel_list.append(child)
1407
+ if "cibling" in self.config.relations:
1408
+ rel_list.append(cibling)
1409
+
1410
+ rel = torch.stack(rel_list, dim=1)
1411
+
1412
+ if n_cntxt_layers > 0:
1413
+ if n_cntxt_layers == 1:
1414
+ rel_weight = self.rel_weight_1
1415
+ elif n_cntxt_layers == 2:
1416
+ rel_weight = self.rel_weight_2
1417
+ else:
1418
+ rel_weight = self.rel_weight
1419
+
1420
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1421
+
1422
+ if n_cntxt_layers == 1:
1423
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1424
+ else:
1425
+ num_layers = self.roberta.config.num_hidden_layers
1426
+
1427
+ att_mask = dep.reshape(
1428
+ num_layers,
1429
+ bsz,
1430
+ self.config.num_attention_heads,
1431
+ length,
1432
+ length,
1433
+ )
1434
+
1435
+ return att_mask, cibling, head, block
1436
+
1437
+ def forward(
1438
+ self,
1439
+ input_ids: Optional[torch.LongTensor] = None,
1440
+ attention_mask: Optional[torch.FloatTensor] = None,
1441
+ token_type_ids: Optional[torch.LongTensor] = None,
1442
+ position_ids: Optional[torch.LongTensor] = None,
1443
+ head_mask: Optional[torch.FloatTensor] = None,
1444
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1445
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
1446
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
1447
+ labels: Optional[torch.LongTensor] = None,
1448
+ output_attentions: Optional[bool] = None,
1449
+ output_hidden_states: Optional[bool] = None,
1450
+ return_dict: Optional[bool] = None,
1451
+ ) -> Union[Tuple, MaskedLMOutput]:
1452
+ r"""
1453
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1454
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
1455
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
1456
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
1457
+ kwargs (`Dict[str, any]`, optional, defaults to *{}*):
1458
+ Used to hide legacy arguments that have been deprecated.
1459
+ """
1460
+ return_dict = (
1461
+ return_dict if return_dict is not None else self.config.use_return_dict
1462
+ )
1463
+
1464
+
1465
+ if self.config.n_cntxt_layers > 0:
1466
+ cntxt_outputs = self.cntxt_layers(
1467
+ input_ids,
1468
+ attention_mask=attention_mask,
1469
+ token_type_ids=token_type_ids,
1470
+ position_ids=position_ids,
1471
+ head_mask=head_mask,
1472
+ inputs_embeds=inputs_embeds,
1473
+ encoder_hidden_states=encoder_hidden_states,
1474
+ encoder_attention_mask=encoder_attention_mask,
1475
+ output_attentions=output_attentions,
1476
+ output_hidden_states=output_hidden_states,
1477
+ return_dict=return_dict)
1478
+
1479
+
1480
+ if self.config.n_cntxt_layers_2 > 0:
1481
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1482
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1483
+
1484
+ cntxt_outputs_2 = self.cntxt_layers_2(
1485
+ input_ids,
1486
+ attention_mask=attention_mask,
1487
+ token_type_ids=token_type_ids,
1488
+ position_ids=position_ids,
1489
+ head_mask=head_mask,
1490
+ inputs_embeds=inputs_embeds,
1491
+ encoder_hidden_states=encoder_hidden_states,
1492
+ encoder_attention_mask=encoder_attention_mask,
1493
+ output_attentions=output_attentions,
1494
+ output_hidden_states=output_hidden_states,
1495
+ return_dict=return_dict,
1496
+ parser_att_mask=att_mask_1)
1497
+
1498
+ sequence_output = cntxt_outputs_2[0]
1499
+
1500
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
1501
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
1502
+
1503
+ elif self.config.n_cntxt_layers > 0:
1504
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
1505
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1506
+ else:
1507
+ distance, height = self.parse(input_ids)
1508
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1509
+
1510
+ outputs = self.roberta(
1511
+ input_ids,
1512
+ attention_mask=attention_mask,
1513
+ token_type_ids=token_type_ids,
1514
+ position_ids=position_ids,
1515
+ head_mask=head_mask,
1516
+ inputs_embeds=inputs_embeds,
1517
+ encoder_hidden_states=encoder_hidden_states,
1518
+ encoder_attention_mask=encoder_attention_mask,
1519
+ output_attentions=output_attentions,
1520
+ output_hidden_states=output_hidden_states,
1521
+ return_dict=return_dict,
1522
+ parser_att_mask=att_mask,
1523
+ )
1524
+ sequence_output = outputs[0]
1525
+ prediction_scores = self.lm_head(sequence_output)
1526
+
1527
+ masked_lm_loss = None
1528
+ if labels is not None:
1529
+ loss_fct = CrossEntropyLoss()
1530
+ masked_lm_loss = loss_fct(
1531
+ prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)
1532
+ )
1533
+
1534
+ if not return_dict:
1535
+ output = (prediction_scores,) + outputs[2:]
1536
+ return (
1537
+ ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
1538
+ )
1539
+
1540
+ return MaskedLMOutput(
1541
+ loss=masked_lm_loss,
1542
+ logits=prediction_scores,
1543
+ hidden_states=outputs.hidden_states,
1544
+ attentions=outputs.attentions,
1545
+ )
1546
+
1547
+
1548
+ class RobertaLMHead(nn.Module):
1549
+ """Roberta Head for masked language modeling."""
1550
+
1551
+ def __init__(self, config):
1552
+ super().__init__()
1553
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
1554
+ self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
1555
+
1556
+ self.decoder = nn.Linear(config.hidden_size, config.vocab_size)
1557
+ self.bias = nn.Parameter(torch.zeros(config.vocab_size))
1558
+ self.decoder.bias = self.bias
1559
+
1560
+ def forward(self, features, **kwargs):
1561
+ x = self.dense(features)
1562
+ x = gelu(x)
1563
+ x = self.layer_norm(x)
1564
+
1565
+ # project back to size of vocabulary with bias
1566
+ x = self.decoder(x)
1567
+
1568
+ return x
1569
+
1570
+ def _tie_weights(self):
1571
+ # To tie those two weights if they get disconnected (on TPU or when the bias is resized)
1572
+ self.bias = self.decoder.bias
1573
+
1574
+
1575
+ class StructRobertaForSequenceClassification(RobertaPreTrainedModel):
1576
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
1577
+
1578
+ def __init__(self, config):
1579
+ super().__init__(config)
1580
+ self.num_labels = config.num_labels
1581
+ self.config = config
1582
+
1583
+ if config.n_cntxt_layers > 0:
1584
+ config_cntxt = copy.deepcopy(config)
1585
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1586
+
1587
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1588
+
1589
+ if config.n_cntxt_layers_2 > 0:
1590
+ self.parser_layers_1 = nn.ModuleList(
1591
+ [
1592
+ nn.Sequential(
1593
+ Conv1d(config.hidden_size, config.conv_size),
1594
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1595
+ nn.Tanh(),
1596
+ )
1597
+ for i in range(int(config.n_parser_layers/2))
1598
+ ]
1599
+ )
1600
+
1601
+ self.distance_ff_1 = nn.Sequential(
1602
+ Conv1d(config.hidden_size, 2),
1603
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1604
+ nn.Tanh(),
1605
+ nn.Linear(config.hidden_size, 1),
1606
+ )
1607
+
1608
+ self.height_ff_1 = nn.Sequential(
1609
+ nn.Linear(config.hidden_size, config.hidden_size),
1610
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1611
+ nn.Tanh(),
1612
+ nn.Linear(config.hidden_size, 1),
1613
+ )
1614
+
1615
+ n_rel = len(config.relations)
1616
+ self._rel_weight_1 = nn.Parameter(
1617
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1618
+ )
1619
+ self._rel_weight_1.data.normal_(0, 0.1)
1620
+
1621
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1622
+
1623
+ config_cntxt_2 = copy.deepcopy(config)
1624
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1625
+
1626
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1627
+
1628
+
1629
+ self.parser_layers_2 = nn.ModuleList(
1630
+ [
1631
+ nn.Sequential(
1632
+ Conv1d(config.hidden_size, config.conv_size),
1633
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1634
+ nn.Tanh(),
1635
+ )
1636
+ for i in range(int(config.n_parser_layers/2))
1637
+ ]
1638
+ )
1639
+
1640
+ self.distance_ff_2 = nn.Sequential(
1641
+ Conv1d(config.hidden_size, 2),
1642
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1643
+ nn.Tanh(),
1644
+ nn.Linear(config.hidden_size, 1),
1645
+ )
1646
+
1647
+ self.height_ff_2 = nn.Sequential(
1648
+ nn.Linear(config.hidden_size, config.hidden_size),
1649
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1650
+ nn.Tanh(),
1651
+ nn.Linear(config.hidden_size, 1),
1652
+ )
1653
+
1654
+ n_rel = len(config.relations)
1655
+ self._rel_weight_2 = nn.Parameter(
1656
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1657
+ )
1658
+ self._rel_weight_2.data.normal_(0, 0.1)
1659
+
1660
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1661
+
1662
+ else:
1663
+ self.parser_layers = nn.ModuleList(
1664
+ [
1665
+ nn.Sequential(
1666
+ Conv1d(config.hidden_size, config.conv_size),
1667
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1668
+ nn.Tanh(),
1669
+ )
1670
+ for i in range(config.n_parser_layers)
1671
+ ]
1672
+ )
1673
+
1674
+ self.distance_ff = nn.Sequential(
1675
+ Conv1d(config.hidden_size, 2),
1676
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1677
+ nn.Tanh(),
1678
+ nn.Linear(config.hidden_size, 1),
1679
+ )
1680
+
1681
+ self.height_ff = nn.Sequential(
1682
+ nn.Linear(config.hidden_size, config.hidden_size),
1683
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1684
+ nn.Tanh(),
1685
+ nn.Linear(config.hidden_size, 1),
1686
+ )
1687
+
1688
+ n_rel = len(config.relations)
1689
+ self._rel_weight = nn.Parameter(
1690
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1691
+ )
1692
+ self._rel_weight.data.normal_(0, 0.1)
1693
+
1694
+ self._scaler = nn.Parameter(torch.zeros(2))
1695
+
1696
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1697
+
1698
+ if config.n_cntxt_layers > 0:
1699
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1700
+ if config.n_cntxt_layers_2 > 0:
1701
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1702
+
1703
+
1704
+ self.pad = config.pad_token_id
1705
+ self.classifier = RobertaClassificationHead(config)
1706
+
1707
+ # Initialize weights and apply final processing
1708
+ self.post_init()
1709
+
1710
+
1711
+ @property
1712
+ def scaler(self):
1713
+ return self._scaler.exp()
1714
+
1715
+ @property
1716
+ def scaler_1(self):
1717
+ return self._scaler_1.exp()
1718
+
1719
+ @property
1720
+ def scaler_2(self):
1721
+ return self._scaler_2.exp()
1722
+
1723
+ @property
1724
+ def rel_weight(self):
1725
+ if self.config.weight_act == "sigmoid":
1726
+ return torch.sigmoid(self._rel_weight)
1727
+ elif self.config.weight_act == "softmax":
1728
+ return torch.softmax(self._rel_weight, dim=-1)
1729
+
1730
+ @property
1731
+ def rel_weight_1(self):
1732
+ if self.config.weight_act == "sigmoid":
1733
+ return torch.sigmoid(self._rel_weight_1)
1734
+ elif self.config.weight_act == "softmax":
1735
+ return torch.softmax(self._rel_weight_1, dim=-1)
1736
+
1737
+
1738
+ @property
1739
+ def rel_weight_2(self):
1740
+ if self.config.weight_act == "sigmoid":
1741
+ return torch.sigmoid(self._rel_weight_2)
1742
+ elif self.config.weight_act == "softmax":
1743
+ return torch.softmax(self._rel_weight_2, dim=-1)
1744
+
1745
+
1746
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1747
+ """Compute constituents from distance and height."""
1748
+
1749
+ if n_cntxt_layers>0:
1750
+ if n_cntxt_layers == 1:
1751
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1752
+ elif n_cntxt_layers == 2:
1753
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1754
+ else:
1755
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1756
+
1757
+ gamma = torch.sigmoid(-beta_logits)
1758
+ ones = torch.ones_like(gamma)
1759
+
1760
+ block_mask_left = cummin(
1761
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1762
+ )
1763
+ block_mask_left = block_mask_left - F.pad(
1764
+ block_mask_left[:, :, :-1], (1, 0), value=0
1765
+ )
1766
+ block_mask_left.tril_(0)
1767
+
1768
+ block_mask_right = cummin(
1769
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1770
+ )
1771
+ block_mask_right = block_mask_right - F.pad(
1772
+ block_mask_right[:, :, 1:], (0, 1), value=0
1773
+ )
1774
+ block_mask_right.triu_(0)
1775
+
1776
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1777
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1778
+ block_mask_right, reverse=True
1779
+ ).triu(1)
1780
+
1781
+ return block_p, block
1782
+
1783
+ def compute_head(self, height, n_cntxt_layers=0):
1784
+ """Estimate head for each constituent."""
1785
+
1786
+ _, length = height.size()
1787
+ if n_cntxt_layers>0:
1788
+ if n_cntxt_layers == 1:
1789
+ head_logits = height * self.scaler_1[1]
1790
+ elif n_cntxt_layers == 2:
1791
+ head_logits = height * self.scaler_2[1]
1792
+ else:
1793
+ head_logits = height * self.scaler[1]
1794
+ index = torch.arange(length, device=height.device)
1795
+
1796
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1797
+ index[None, None, :] <= index[None, :, None]
1798
+ )
1799
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1800
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1801
+
1802
+ head_p = torch.softmax(head_logits, dim=-1)
1803
+
1804
+ return head_p
1805
+
1806
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1807
+ """Parse input sentence.
1808
+
1809
+ Args:
1810
+ x: input tokens (required).
1811
+ pos: position for each token (optional).
1812
+ Returns:
1813
+ distance: syntactic distance
1814
+ height: syntactic height
1815
+ """
1816
+
1817
+ mask = x != self.pad
1818
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1819
+
1820
+ if embs is None:
1821
+ h = self.roberta.embeddings(x)
1822
+ else:
1823
+ h = embs
1824
+
1825
+ if n_cntxt_layers > 0:
1826
+ if n_cntxt_layers == 1:
1827
+ parser_layers = self.parser_layers_1
1828
+ height_ff = self.height_ff_1
1829
+ distance_ff = self.distance_ff_1
1830
+ elif n_cntxt_layers == 2:
1831
+ parser_layers = self.parser_layers_2
1832
+ height_ff = self.height_ff_2
1833
+ distance_ff = self.distance_ff_2
1834
+ for i in range(int(self.config.n_parser_layers/2)):
1835
+ h = h.masked_fill(~mask[:, :, None], 0)
1836
+ h = parser_layers[i](h)
1837
+
1838
+ height = height_ff(h).squeeze(-1)
1839
+ height.masked_fill_(~mask, -1e9)
1840
+
1841
+ distance = distance_ff(h).squeeze(-1)
1842
+ distance.masked_fill_(~mask_shifted, 1e9)
1843
+
1844
+ # Calbrating the distance and height to the same level
1845
+ length = distance.size(1)
1846
+ height_max = height[:, None, :].expand(-1, length, -1)
1847
+ height_max = torch.cummax(
1848
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1849
+ )[0].triu(0)
1850
+
1851
+ margin_left = torch.relu(
1852
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1853
+ )
1854
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1855
+ margin = torch.where(
1856
+ margin_left > margin_right, margin_right, margin_left
1857
+ ).triu(0)
1858
+
1859
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1860
+ margin.masked_fill_(~margin_mask, 0)
1861
+ margin = margin.max()
1862
+
1863
+ distance = distance - margin
1864
+ else:
1865
+ for i in range(self.config.n_parser_layers):
1866
+ h = h.masked_fill(~mask[:, :, None], 0)
1867
+ h = self.parser_layers[i](h)
1868
+
1869
+ height = self.height_ff(h).squeeze(-1)
1870
+ height.masked_fill_(~mask, -1e9)
1871
+
1872
+ distance = self.distance_ff(h).squeeze(-1)
1873
+ distance.masked_fill_(~mask_shifted, 1e9)
1874
+
1875
+ # Calbrating the distance and height to the same level
1876
+ length = distance.size(1)
1877
+ height_max = height[:, None, :].expand(-1, length, -1)
1878
+ height_max = torch.cummax(
1879
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1880
+ )[0].triu(0)
1881
+
1882
+ margin_left = torch.relu(
1883
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1884
+ )
1885
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1886
+ margin = torch.where(
1887
+ margin_left > margin_right, margin_right, margin_left
1888
+ ).triu(0)
1889
+
1890
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1891
+ margin.masked_fill_(~margin_mask, 0)
1892
+ margin = margin.max()
1893
+
1894
+ distance = distance - margin
1895
+
1896
+ return distance, height
1897
+
1898
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1899
+ """Compute head and cibling distribution for each token."""
1900
+
1901
+ bsz, length = x.size()
1902
+
1903
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1904
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1905
+
1906
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1907
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1908
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1909
+ head = head.masked_fill(eye, 0)
1910
+ child = head.transpose(1, 2)
1911
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1912
+
1913
+ rel_list = []
1914
+ if "head" in self.config.relations:
1915
+ rel_list.append(head)
1916
+ if "child" in self.config.relations:
1917
+ rel_list.append(child)
1918
+ if "cibling" in self.config.relations:
1919
+ rel_list.append(cibling)
1920
+
1921
+ rel = torch.stack(rel_list, dim=1)
1922
+
1923
+ if n_cntxt_layers > 0:
1924
+ if n_cntxt_layers == 1:
1925
+ rel_weight = self.rel_weight_1
1926
+ elif n_cntxt_layers == 2:
1927
+ rel_weight = self.rel_weight_2
1928
+ else:
1929
+ rel_weight = self.rel_weight
1930
+
1931
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1932
+
1933
+ if n_cntxt_layers == 1:
1934
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1935
+ else:
1936
+ num_layers = self.roberta.config.num_hidden_layers
1937
+
1938
+ att_mask = dep.reshape(
1939
+ num_layers,
1940
+ bsz,
1941
+ self.config.num_attention_heads,
1942
+ length,
1943
+ length,
1944
+ )
1945
+
1946
+ return att_mask, cibling, head, block
1947
+
1948
+ def forward(
1949
+ self,
1950
+ input_ids: Optional[torch.LongTensor] = None,
1951
+ attention_mask: Optional[torch.FloatTensor] = None,
1952
+ token_type_ids: Optional[torch.LongTensor] = None,
1953
+ position_ids: Optional[torch.LongTensor] = None,
1954
+ head_mask: Optional[torch.FloatTensor] = None,
1955
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1956
+ labels: Optional[torch.LongTensor] = None,
1957
+ output_attentions: Optional[bool] = None,
1958
+ output_hidden_states: Optional[bool] = None,
1959
+ return_dict: Optional[bool] = None,
1960
+ ) -> Union[Tuple, SequenceClassifierOutput]:
1961
+ r"""
1962
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1963
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1964
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1965
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1966
+ """
1967
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1968
+
1969
+ if self.config.n_cntxt_layers > 0:
1970
+ cntxt_outputs = self.cntxt_layers(
1971
+ input_ids,
1972
+ attention_mask=attention_mask,
1973
+ token_type_ids=token_type_ids,
1974
+ position_ids=position_ids,
1975
+ head_mask=head_mask,
1976
+ inputs_embeds=inputs_embeds,
1977
+ output_attentions=output_attentions,
1978
+ output_hidden_states=output_hidden_states,
1979
+ return_dict=return_dict)
1980
+
1981
+
1982
+ if self.config.n_cntxt_layers_2 > 0:
1983
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1984
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1985
+
1986
+ cntxt_outputs_2 = self.cntxt_layers_2(
1987
+ input_ids,
1988
+ attention_mask=attention_mask,
1989
+ token_type_ids=token_type_ids,
1990
+ position_ids=position_ids,
1991
+ head_mask=head_mask,
1992
+ inputs_embeds=inputs_embeds,
1993
+ output_attentions=output_attentions,
1994
+ output_hidden_states=output_hidden_states,
1995
+ return_dict=return_dict,
1996
+ parser_att_mask=att_mask_1)
1997
+
1998
+ sequence_output = cntxt_outputs_2[0]
1999
+
2000
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
2001
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
2002
+
2003
+ elif self.config.n_cntxt_layers > 0:
2004
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
2005
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2006
+ else:
2007
+ distance, height = self.parse(input_ids)
2008
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2009
+
2010
+ outputs = self.roberta(
2011
+ input_ids,
2012
+ attention_mask=attention_mask,
2013
+ token_type_ids=token_type_ids,
2014
+ position_ids=position_ids,
2015
+ head_mask=head_mask,
2016
+ inputs_embeds=inputs_embeds,
2017
+ output_attentions=output_attentions,
2018
+ output_hidden_states=output_hidden_states,
2019
+ return_dict=return_dict,
2020
+ parser_att_mask=att_mask,
2021
+ )
2022
+ sequence_output = outputs[0]
2023
+ logits = self.classifier(sequence_output)
2024
+
2025
+ loss = None
2026
+ if labels is not None:
2027
+ if self.config.problem_type is None:
2028
+ if self.num_labels == 1:
2029
+ self.config.problem_type = "regression"
2030
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
2031
+ self.config.problem_type = "single_label_classification"
2032
+ else:
2033
+ self.config.problem_type = "multi_label_classification"
2034
+
2035
+ if self.config.problem_type == "regression":
2036
+ loss_fct = MSELoss()
2037
+ if self.num_labels == 1:
2038
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
2039
+ else:
2040
+ loss = loss_fct(logits, labels)
2041
+ elif self.config.problem_type == "single_label_classification":
2042
+ loss_fct = CrossEntropyLoss()
2043
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
2044
+ elif self.config.problem_type == "multi_label_classification":
2045
+ loss_fct = BCEWithLogitsLoss()
2046
+ loss = loss_fct(logits, labels)
2047
+
2048
+ if not return_dict:
2049
+ output = (logits,) + outputs[2:]
2050
+ return ((loss,) + output) if loss is not None else output
2051
+
2052
+ return SequenceClassifierOutput(
2053
+ loss=loss,
2054
+ logits=logits,
2055
+ hidden_states=outputs.hidden_states,
2056
+ attentions=outputs.attentions,
2057
+ )
2058
+
2059
+
2060
+ class RobertaClassificationHead(nn.Module):
2061
+ """Head for sentence-level classification tasks."""
2062
+
2063
+ def __init__(self, config):
2064
+ super().__init__()
2065
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
2066
+ classifier_dropout = (
2067
+ config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
2068
+ )
2069
+ self.dropout = nn.Dropout(classifier_dropout)
2070
+ self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
2071
+
2072
+ def forward(self, features, **kwargs):
2073
+ x = features[:, 0, :] # take <s> token (equiv. to [CLS])
2074
+ x = self.dropout(x)
2075
+ x = self.dense(x)
2076
+ x = torch.tanh(x)
2077
+ x = self.dropout(x)
2078
+ x = self.out_proj(x)
2079
+ return x
2080
+
2081
+
2082
+ def create_position_ids_from_input_ids(
2083
+ input_ids, padding_idx, past_key_values_length=0
2084
+ ):
2085
+ """
2086
+ Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
2087
+ are ignored. This is modified from fairseq's `utils.make_positions`.
2088
+
2089
+ Args:
2090
+ x: torch.Tensor x:
2091
+
2092
+ Returns: torch.Tensor
2093
+ """
2094
+ # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
2095
+ mask = input_ids.ne(padding_idx).int()
2096
+ incremental_indices = (
2097
+ torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length
2098
+ ) * mask
2099
+ return incremental_indices.long() + padding_idx
2100
+
2101
+
2102
+ def cumprod(x, reverse=False, exclusive=False):
2103
+ """cumulative product."""
2104
+ if reverse:
2105
+ x = x.flip([-1])
2106
+
2107
+ if exclusive:
2108
+ x = F.pad(x[:, :, :-1], (1, 0), value=1)
2109
+
2110
+ cx = x.cumprod(-1)
2111
+
2112
+ if reverse:
2113
+ cx = cx.flip([-1])
2114
+ return cx
2115
+
2116
+
2117
+ def cumsum(x, reverse=False, exclusive=False):
2118
+ """cumulative sum."""
2119
+ bsz, _, length = x.size()
2120
+ device = x.device
2121
+ if reverse:
2122
+ if exclusive:
2123
+ w = torch.ones([bsz, length, length], device=device).tril(-1)
2124
+ else:
2125
+ w = torch.ones([bsz, length, length], device=device).tril(0)
2126
+ cx = torch.bmm(x, w)
2127
+ else:
2128
+ if exclusive:
2129
+ w = torch.ones([bsz, length, length], device=device).triu(1)
2130
+ else:
2131
+ w = torch.ones([bsz, length, length], device=device).triu(0)
2132
+ cx = torch.bmm(x, w)
2133
+ return cx
2134
+
2135
+
2136
+ def cummin(x, reverse=False, exclusive=False, max_value=1e9):
2137
+ """cumulative min."""
2138
+ if reverse:
2139
+ if exclusive:
2140
+ x = F.pad(x[:, :, 1:], (0, 1), value=max_value)
2141
+ x = x.flip([-1]).cummin(-1)[0].flip([-1])
2142
+ else:
2143
+ if exclusive:
2144
+ x = F.pad(x[:, :, :-1], (1, 0), value=max_value)
2145
+ x = x.cummin(-1)[0]
2146
+ return x
finetune/boolq/predict_results.txt ADDED
@@ -0,0 +1,724 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ index prediction
2
+ 0 1
3
+ 1 1
4
+ 2 1
5
+ 3 1
6
+ 4 0
7
+ 5 1
8
+ 6 1
9
+ 7 0
10
+ 8 1
11
+ 9 0
12
+ 10 1
13
+ 11 1
14
+ 12 0
15
+ 13 1
16
+ 14 1
17
+ 15 1
18
+ 16 1
19
+ 17 1
20
+ 18 1
21
+ 19 1
22
+ 20 0
23
+ 21 1
24
+ 22 1
25
+ 23 1
26
+ 24 1
27
+ 25 1
28
+ 26 0
29
+ 27 1
30
+ 28 0
31
+ 29 1
32
+ 30 1
33
+ 31 1
34
+ 32 1
35
+ 33 1
36
+ 34 1
37
+ 35 1
38
+ 36 0
39
+ 37 1
40
+ 38 1
41
+ 39 1
42
+ 40 1
43
+ 41 0
44
+ 42 1
45
+ 43 1
46
+ 44 0
47
+ 45 1
48
+ 46 1
49
+ 47 1
50
+ 48 1
51
+ 49 1
52
+ 50 0
53
+ 51 1
54
+ 52 0
55
+ 53 1
56
+ 54 0
57
+ 55 1
58
+ 56 1
59
+ 57 1
60
+ 58 1
61
+ 59 0
62
+ 60 0
63
+ 61 1
64
+ 62 1
65
+ 63 1
66
+ 64 1
67
+ 65 1
68
+ 66 1
69
+ 67 1
70
+ 68 1
71
+ 69 1
72
+ 70 1
73
+ 71 1
74
+ 72 0
75
+ 73 1
76
+ 74 0
77
+ 75 1
78
+ 76 1
79
+ 77 0
80
+ 78 1
81
+ 79 0
82
+ 80 1
83
+ 81 0
84
+ 82 0
85
+ 83 1
86
+ 84 1
87
+ 85 1
88
+ 86 0
89
+ 87 1
90
+ 88 0
91
+ 89 1
92
+ 90 1
93
+ 91 1
94
+ 92 1
95
+ 93 1
96
+ 94 0
97
+ 95 1
98
+ 96 1
99
+ 97 1
100
+ 98 1
101
+ 99 1
102
+ 100 1
103
+ 101 0
104
+ 102 1
105
+ 103 0
106
+ 104 0
107
+ 105 1
108
+ 106 1
109
+ 107 0
110
+ 108 1
111
+ 109 1
112
+ 110 1
113
+ 111 1
114
+ 112 1
115
+ 113 0
116
+ 114 1
117
+ 115 1
118
+ 116 1
119
+ 117 1
120
+ 118 1
121
+ 119 0
122
+ 120 1
123
+ 121 0
124
+ 122 1
125
+ 123 0
126
+ 124 0
127
+ 125 1
128
+ 126 0
129
+ 127 1
130
+ 128 1
131
+ 129 1
132
+ 130 0
133
+ 131 0
134
+ 132 0
135
+ 133 0
136
+ 134 1
137
+ 135 1
138
+ 136 1
139
+ 137 1
140
+ 138 1
141
+ 139 0
142
+ 140 1
143
+ 141 1
144
+ 142 1
145
+ 143 1
146
+ 144 0
147
+ 145 1
148
+ 146 1
149
+ 147 0
150
+ 148 1
151
+ 149 1
152
+ 150 1
153
+ 151 1
154
+ 152 1
155
+ 153 1
156
+ 154 1
157
+ 155 0
158
+ 156 1
159
+ 157 1
160
+ 158 0
161
+ 159 1
162
+ 160 1
163
+ 161 1
164
+ 162 1
165
+ 163 0
166
+ 164 1
167
+ 165 1
168
+ 166 1
169
+ 167 1
170
+ 168 0
171
+ 169 1
172
+ 170 1
173
+ 171 1
174
+ 172 0
175
+ 173 1
176
+ 174 1
177
+ 175 1
178
+ 176 1
179
+ 177 1
180
+ 178 0
181
+ 179 1
182
+ 180 1
183
+ 181 1
184
+ 182 1
185
+ 183 0
186
+ 184 0
187
+ 185 1
188
+ 186 0
189
+ 187 0
190
+ 188 0
191
+ 189 1
192
+ 190 1
193
+ 191 1
194
+ 192 1
195
+ 193 0
196
+ 194 1
197
+ 195 0
198
+ 196 1
199
+ 197 1
200
+ 198 0
201
+ 199 1
202
+ 200 0
203
+ 201 1
204
+ 202 1
205
+ 203 0
206
+ 204 1
207
+ 205 1
208
+ 206 1
209
+ 207 1
210
+ 208 1
211
+ 209 1
212
+ 210 1
213
+ 211 1
214
+ 212 1
215
+ 213 1
216
+ 214 1
217
+ 215 1
218
+ 216 1
219
+ 217 1
220
+ 218 1
221
+ 219 0
222
+ 220 1
223
+ 221 1
224
+ 222 1
225
+ 223 0
226
+ 224 0
227
+ 225 0
228
+ 226 1
229
+ 227 1
230
+ 228 0
231
+ 229 1
232
+ 230 0
233
+ 231 1
234
+ 232 1
235
+ 233 0
236
+ 234 1
237
+ 235 1
238
+ 236 0
239
+ 237 1
240
+ 238 1
241
+ 239 1
242
+ 240 1
243
+ 241 1
244
+ 242 1
245
+ 243 1
246
+ 244 0
247
+ 245 1
248
+ 246 1
249
+ 247 0
250
+ 248 1
251
+ 249 1
252
+ 250 1
253
+ 251 0
254
+ 252 0
255
+ 253 1
256
+ 254 1
257
+ 255 0
258
+ 256 1
259
+ 257 1
260
+ 258 1
261
+ 259 0
262
+ 260 1
263
+ 261 0
264
+ 262 1
265
+ 263 1
266
+ 264 1
267
+ 265 1
268
+ 266 1
269
+ 267 0
270
+ 268 1
271
+ 269 1
272
+ 270 1
273
+ 271 1
274
+ 272 1
275
+ 273 1
276
+ 274 1
277
+ 275 1
278
+ 276 1
279
+ 277 1
280
+ 278 0
281
+ 279 0
282
+ 280 0
283
+ 281 1
284
+ 282 0
285
+ 283 0
286
+ 284 1
287
+ 285 1
288
+ 286 0
289
+ 287 0
290
+ 288 1
291
+ 289 0
292
+ 290 1
293
+ 291 0
294
+ 292 1
295
+ 293 1
296
+ 294 1
297
+ 295 1
298
+ 296 0
299
+ 297 0
300
+ 298 1
301
+ 299 1
302
+ 300 1
303
+ 301 0
304
+ 302 0
305
+ 303 0
306
+ 304 1
307
+ 305 0
308
+ 306 1
309
+ 307 0
310
+ 308 1
311
+ 309 0
312
+ 310 0
313
+ 311 1
314
+ 312 1
315
+ 313 1
316
+ 314 1
317
+ 315 1
318
+ 316 1
319
+ 317 1
320
+ 318 1
321
+ 319 1
322
+ 320 1
323
+ 321 1
324
+ 322 1
325
+ 323 1
326
+ 324 1
327
+ 325 0
328
+ 326 1
329
+ 327 1
330
+ 328 1
331
+ 329 1
332
+ 330 0
333
+ 331 1
334
+ 332 1
335
+ 333 1
336
+ 334 0
337
+ 335 1
338
+ 336 1
339
+ 337 1
340
+ 338 0
341
+ 339 1
342
+ 340 1
343
+ 341 1
344
+ 342 1
345
+ 343 1
346
+ 344 1
347
+ 345 1
348
+ 346 1
349
+ 347 1
350
+ 348 1
351
+ 349 1
352
+ 350 1
353
+ 351 1
354
+ 352 1
355
+ 353 1
356
+ 354 1
357
+ 355 0
358
+ 356 0
359
+ 357 1
360
+ 358 1
361
+ 359 0
362
+ 360 1
363
+ 361 0
364
+ 362 0
365
+ 363 1
366
+ 364 1
367
+ 365 1
368
+ 366 0
369
+ 367 1
370
+ 368 1
371
+ 369 1
372
+ 370 1
373
+ 371 1
374
+ 372 1
375
+ 373 1
376
+ 374 1
377
+ 375 1
378
+ 376 1
379
+ 377 1
380
+ 378 0
381
+ 379 0
382
+ 380 0
383
+ 381 0
384
+ 382 1
385
+ 383 0
386
+ 384 1
387
+ 385 0
388
+ 386 1
389
+ 387 1
390
+ 388 1
391
+ 389 1
392
+ 390 1
393
+ 391 0
394
+ 392 1
395
+ 393 1
396
+ 394 1
397
+ 395 1
398
+ 396 1
399
+ 397 1
400
+ 398 1
401
+ 399 0
402
+ 400 1
403
+ 401 0
404
+ 402 1
405
+ 403 1
406
+ 404 1
407
+ 405 1
408
+ 406 0
409
+ 407 0
410
+ 408 1
411
+ 409 0
412
+ 410 1
413
+ 411 0
414
+ 412 0
415
+ 413 1
416
+ 414 0
417
+ 415 0
418
+ 416 1
419
+ 417 0
420
+ 418 0
421
+ 419 1
422
+ 420 1
423
+ 421 1
424
+ 422 1
425
+ 423 0
426
+ 424 1
427
+ 425 0
428
+ 426 1
429
+ 427 1
430
+ 428 0
431
+ 429 1
432
+ 430 1
433
+ 431 0
434
+ 432 1
435
+ 433 1
436
+ 434 1
437
+ 435 1
438
+ 436 1
439
+ 437 1
440
+ 438 1
441
+ 439 1
442
+ 440 0
443
+ 441 1
444
+ 442 0
445
+ 443 1
446
+ 444 0
447
+ 445 1
448
+ 446 1
449
+ 447 1
450
+ 448 1
451
+ 449 1
452
+ 450 1
453
+ 451 1
454
+ 452 1
455
+ 453 0
456
+ 454 1
457
+ 455 0
458
+ 456 1
459
+ 457 1
460
+ 458 1
461
+ 459 1
462
+ 460 1
463
+ 461 1
464
+ 462 1
465
+ 463 0
466
+ 464 0
467
+ 465 0
468
+ 466 1
469
+ 467 1
470
+ 468 1
471
+ 469 0
472
+ 470 1
473
+ 471 1
474
+ 472 1
475
+ 473 1
476
+ 474 0
477
+ 475 0
478
+ 476 1
479
+ 477 0
480
+ 478 0
481
+ 479 1
482
+ 480 0
483
+ 481 1
484
+ 482 0
485
+ 483 1
486
+ 484 1
487
+ 485 1
488
+ 486 1
489
+ 487 1
490
+ 488 1
491
+ 489 0
492
+ 490 1
493
+ 491 1
494
+ 492 1
495
+ 493 0
496
+ 494 1
497
+ 495 1
498
+ 496 1
499
+ 497 0
500
+ 498 0
501
+ 499 1
502
+ 500 1
503
+ 501 0
504
+ 502 0
505
+ 503 1
506
+ 504 1
507
+ 505 1
508
+ 506 1
509
+ 507 1
510
+ 508 0
511
+ 509 1
512
+ 510 1
513
+ 511 1
514
+ 512 1
515
+ 513 1
516
+ 514 0
517
+ 515 0
518
+ 516 1
519
+ 517 1
520
+ 518 1
521
+ 519 1
522
+ 520 0
523
+ 521 1
524
+ 522 0
525
+ 523 1
526
+ 524 1
527
+ 525 1
528
+ 526 0
529
+ 527 1
530
+ 528 0
531
+ 529 0
532
+ 530 1
533
+ 531 0
534
+ 532 1
535
+ 533 1
536
+ 534 0
537
+ 535 1
538
+ 536 1
539
+ 537 0
540
+ 538 1
541
+ 539 0
542
+ 540 1
543
+ 541 1
544
+ 542 1
545
+ 543 1
546
+ 544 1
547
+ 545 1
548
+ 546 1
549
+ 547 1
550
+ 548 1
551
+ 549 1
552
+ 550 1
553
+ 551 0
554
+ 552 0
555
+ 553 1
556
+ 554 0
557
+ 555 0
558
+ 556 1
559
+ 557 0
560
+ 558 0
561
+ 559 1
562
+ 560 1
563
+ 561 1
564
+ 562 1
565
+ 563 1
566
+ 564 1
567
+ 565 0
568
+ 566 0
569
+ 567 0
570
+ 568 1
571
+ 569 1
572
+ 570 1
573
+ 571 1
574
+ 572 1
575
+ 573 1
576
+ 574 1
577
+ 575 1
578
+ 576 1
579
+ 577 1
580
+ 578 1
581
+ 579 1
582
+ 580 1
583
+ 581 0
584
+ 582 1
585
+ 583 1
586
+ 584 1
587
+ 585 0
588
+ 586 1
589
+ 587 1
590
+ 588 0
591
+ 589 1
592
+ 590 1
593
+ 591 1
594
+ 592 0
595
+ 593 1
596
+ 594 1
597
+ 595 0
598
+ 596 1
599
+ 597 0
600
+ 598 1
601
+ 599 1
602
+ 600 1
603
+ 601 1
604
+ 602 1
605
+ 603 1
606
+ 604 1
607
+ 605 0
608
+ 606 1
609
+ 607 0
610
+ 608 1
611
+ 609 1
612
+ 610 1
613
+ 611 1
614
+ 612 1
615
+ 613 1
616
+ 614 1
617
+ 615 1
618
+ 616 1
619
+ 617 1
620
+ 618 1
621
+ 619 1
622
+ 620 1
623
+ 621 1
624
+ 622 1
625
+ 623 1
626
+ 624 1
627
+ 625 0
628
+ 626 0
629
+ 627 1
630
+ 628 1
631
+ 629 0
632
+ 630 1
633
+ 631 1
634
+ 632 0
635
+ 633 1
636
+ 634 1
637
+ 635 0
638
+ 636 1
639
+ 637 1
640
+ 638 1
641
+ 639 1
642
+ 640 1
643
+ 641 1
644
+ 642 1
645
+ 643 1
646
+ 644 0
647
+ 645 1
648
+ 646 1
649
+ 647 0
650
+ 648 1
651
+ 649 1
652
+ 650 1
653
+ 651 1
654
+ 652 1
655
+ 653 0
656
+ 654 1
657
+ 655 1
658
+ 656 1
659
+ 657 0
660
+ 658 1
661
+ 659 1
662
+ 660 0
663
+ 661 0
664
+ 662 1
665
+ 663 1
666
+ 664 1
667
+ 665 1
668
+ 666 1
669
+ 667 1
670
+ 668 0
671
+ 669 1
672
+ 670 1
673
+ 671 1
674
+ 672 1
675
+ 673 1
676
+ 674 0
677
+ 675 1
678
+ 676 1
679
+ 677 1
680
+ 678 1
681
+ 679 1
682
+ 680 0
683
+ 681 1
684
+ 682 1
685
+ 683 0
686
+ 684 1
687
+ 685 1
688
+ 686 0
689
+ 687 1
690
+ 688 1
691
+ 689 1
692
+ 690 1
693
+ 691 0
694
+ 692 0
695
+ 693 0
696
+ 694 1
697
+ 695 1
698
+ 696 1
699
+ 697 1
700
+ 698 1
701
+ 699 0
702
+ 700 0
703
+ 701 1
704
+ 702 1
705
+ 703 1
706
+ 704 1
707
+ 705 1
708
+ 706 1
709
+ 707 1
710
+ 708 1
711
+ 709 0
712
+ 710 1
713
+ 711 0
714
+ 712 0
715
+ 713 0
716
+ 714 0
717
+ 715 1
718
+ 716 0
719
+ 717 0
720
+ 718 1
721
+ 719 1
722
+ 720 1
723
+ 721 1
724
+ 722 1
finetune/boolq/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b42ad7d66be4e8b3f1c3df1892a72084cf423773f1b5d6f37a2c9b22410a6f5
3
+ size 534594825
finetune/boolq/special_tokens_map.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "mask_token": {
6
+ "content": "<mask>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "pad_token": "<pad>",
13
+ "sep_token": "</s>",
14
+ "unk_token": "<unk>"
15
+ }
finetune/boolq/tokenizer_config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": {
4
+ "__type": "AddedToken",
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ },
11
+ "cls_token": {
12
+ "__type": "AddedToken",
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "errors": "replace",
28
+ "mask_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<mask>",
31
+ "lstrip": true,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false
35
+ },
36
+ "model_max_length": 512,
37
+ "name_or_path": "final_models/glue_models/structroberta_s2_final/",
38
+ "pad_token": {
39
+ "__type": "AddedToken",
40
+ "content": "<pad>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": false
45
+ },
46
+ "sep_token": {
47
+ "__type": "AddedToken",
48
+ "content": "</s>",
49
+ "lstrip": false,
50
+ "normalized": true,
51
+ "rstrip": false,
52
+ "single_word": false
53
+ },
54
+ "special_tokens_map_file": null,
55
+ "tokenizer_class": "RobertaTokenizer",
56
+ "trim_offsets": true,
57
+ "unk_token": {
58
+ "__type": "AddedToken",
59
+ "content": "<unk>",
60
+ "lstrip": false,
61
+ "normalized": true,
62
+ "rstrip": false,
63
+ "single_word": false
64
+ }
65
+ }
finetune/boolq/train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "train_loss": 0.311509641011556,
4
+ "train_runtime": 90.5546,
5
+ "train_samples": 2072,
6
+ "train_samples_per_second": 228.812,
7
+ "train_steps_per_second": 1.988
8
+ }
finetune/boolq/trainer_state.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 10.0,
5
+ "global_step": 180,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 10.0,
12
+ "step": 180,
13
+ "total_flos": 1729064137973760.0,
14
+ "train_loss": 0.311509641011556,
15
+ "train_runtime": 90.5546,
16
+ "train_samples_per_second": 228.812,
17
+ "train_steps_per_second": 1.988
18
+ }
19
+ ],
20
+ "max_steps": 180,
21
+ "num_train_epochs": 10,
22
+ "total_flos": 1729064137973760.0,
23
+ "trial_name": null,
24
+ "trial_params": null
25
+ }
finetune/boolq/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b78adc9e31abc6f6858f93602b32e5b82e933bb852e171a858f159a326eed73e
3
+ size 3503
finetune/boolq/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
finetune/cola/all_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.7006869316101074,
4
+ "eval_f1": 0.8002619515389654,
5
+ "eval_loss": 0.9536450505256653,
6
+ "eval_mcc": 0.22514041055870862,
7
+ "eval_runtime": 2.0126,
8
+ "eval_samples": 1019,
9
+ "eval_samples_per_second": 506.308,
10
+ "eval_steps_per_second": 63.599,
11
+ "train_loss": 0.24862313063248345,
12
+ "train_runtime": 353.5006,
13
+ "train_samples": 8164,
14
+ "train_samples_per_second": 230.947,
15
+ "train_steps_per_second": 1.952
16
+ }
finetune/cola/config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "final_models/glue_models/structroberta_s2_final/",
3
+ "architectures": [
4
+ "StructRobertaForSequenceClassification"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "auto_map": {
8
+ "AutoConfig": "modeling_structroberta.StructRobertaConfig",
9
+ "AutoModelForMaskedLM": "modeling_structroberta.StructRoberta",
10
+ "AutoModelForSequenceClassification": "modeling_structroberta.StructRobertaForSequenceClassification"
11
+ },
12
+ "bos_token_id": 0,
13
+ "classifier_dropout": null,
14
+ "conv_size": 9,
15
+ "eos_token_id": 2,
16
+ "hidden_act": "gelu",
17
+ "hidden_dropout_prob": 0.1,
18
+ "hidden_size": 768,
19
+ "id2label": {
20
+ "0": 0,
21
+ "1": 1
22
+ },
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 3072,
25
+ "label2id": {
26
+ "0": 0,
27
+ "1": 1
28
+ },
29
+ "layer_norm_eps": 1e-05,
30
+ "max_position_embeddings": 514,
31
+ "model_type": "roberta",
32
+ "n_cntxt_layers": 4,
33
+ "n_cntxt_layers_2": 0,
34
+ "n_parser_layers": 4,
35
+ "num_attention_heads": 12,
36
+ "num_hidden_layers": 8,
37
+ "pad_token_id": 1,
38
+ "position_embedding_type": "absolute",
39
+ "problem_type": "single_label_classification",
40
+ "relations": [
41
+ "head",
42
+ "child"
43
+ ],
44
+ "torch_dtype": "float32",
45
+ "transformers_version": "4.26.1",
46
+ "type_vocab_size": 1,
47
+ "use_cache": true,
48
+ "vocab_size": 32000,
49
+ "weight_act": "softmax"
50
+ }
finetune/cola/eval_results.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.7006869316101074,
4
+ "eval_f1": 0.8002619515389654,
5
+ "eval_loss": 0.9536450505256653,
6
+ "eval_mcc": 0.22514041055870862,
7
+ "eval_runtime": 2.0126,
8
+ "eval_samples": 1019,
9
+ "eval_samples_per_second": 506.308,
10
+ "eval_steps_per_second": 63.599
11
+ }
finetune/cola/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
finetune/cola/modeling_structroberta.py ADDED
@@ -0,0 +1,2146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import math
3
+ from typing import List, Optional, Tuple, Union
4
+ import torch
5
+ import torch.utils.checkpoint
6
+ from packaging import version
7
+ from torch import nn
8
+ import torch.nn.functional as F
9
+ from torch.nn import CrossEntropyLoss, MSELoss, BCEWithLogitsLoss
10
+ from transformers.activations import ACT2FN, gelu
11
+ from transformers.modeling_outputs import (
12
+ BaseModelOutputWithPastAndCrossAttentions,
13
+ BaseModelOutputWithPoolingAndCrossAttentions,
14
+ MaskedLMOutput,
15
+ SequenceClassifierOutput
16
+ )
17
+ from transformers.modeling_utils import (
18
+ PreTrainedModel,
19
+ apply_chunking_to_forward,
20
+ find_pruneable_heads_and_indices,
21
+ prune_linear_layer,
22
+ )
23
+ from transformers.utils import logging
24
+ from transformers import RobertaConfig
25
+
26
+
27
+ logger = logging.get_logger(__name__)
28
+ ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [
29
+ "roberta-base",
30
+ "roberta-large",
31
+ "roberta-large-mnli",
32
+ "distilroberta-base",
33
+ "roberta-base-openai-detector",
34
+ "roberta-large-openai-detector",
35
+ # See all RoBERTa models at https://huggingface.co/models?filter=roberta
36
+ ]
37
+
38
+
39
+ class StructRobertaConfig(RobertaConfig):
40
+ model_type = "roberta"
41
+
42
+ def __init__(
43
+ self,
44
+ n_parser_layers=4,
45
+ conv_size=9,
46
+ relations=("head", "child"),
47
+ weight_act="softmax",
48
+ n_cntxt_layers=3,
49
+ n_cntxt_layers_2=0,
50
+ **kwargs,):
51
+
52
+ super().__init__(**kwargs)
53
+ self.n_cntxt_layers = n_cntxt_layers
54
+ self.n_parser_layers = n_parser_layers
55
+ self.n_cntxt_layers_2 = n_cntxt_layers_2
56
+ self.conv_size = conv_size
57
+ self.relations = relations
58
+ self.weight_act = weight_act
59
+
60
+ class Conv1d(nn.Module):
61
+ """1D convolution layer."""
62
+
63
+ def __init__(self, hidden_size, kernel_size, dilation=1):
64
+ """Initialization.
65
+
66
+ Args:
67
+ hidden_size: dimension of input embeddings
68
+ kernel_size: convolution kernel size
69
+ dilation: the spacing between the kernel points
70
+ """
71
+ super(Conv1d, self).__init__()
72
+
73
+ if kernel_size % 2 == 0:
74
+ padding = (kernel_size // 2) * dilation
75
+ self.shift = True
76
+ else:
77
+ padding = ((kernel_size - 1) // 2) * dilation
78
+ self.shift = False
79
+ self.conv = nn.Conv1d(
80
+ hidden_size, hidden_size, kernel_size, padding=padding, dilation=dilation
81
+ )
82
+
83
+ def forward(self, x):
84
+ """Compute convolution.
85
+
86
+ Args:
87
+ x: input embeddings
88
+ Returns:
89
+ conv_output: convolution results
90
+ """
91
+
92
+ if self.shift:
93
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)[:, 1:]
94
+ else:
95
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)
96
+
97
+
98
+ class RobertaEmbeddings(nn.Module):
99
+ """
100
+ Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
101
+ """
102
+
103
+ # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__
104
+ def __init__(self, config):
105
+ super().__init__()
106
+ self.word_embeddings = nn.Embedding(
107
+ config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id
108
+ )
109
+ self.position_embeddings = nn.Embedding(
110
+ config.max_position_embeddings, config.hidden_size
111
+ )
112
+ self.token_type_embeddings = nn.Embedding(
113
+ config.type_vocab_size, config.hidden_size
114
+ )
115
+
116
+ # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
117
+ # any TensorFlow checkpoint file
118
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
119
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
120
+ # position_ids (1, len position emb) is contiguous in memory and exported when serialized
121
+ self.position_embedding_type = getattr(
122
+ config, "position_embedding_type", "absolute"
123
+ )
124
+ self.register_buffer(
125
+ "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))
126
+ )
127
+ if version.parse(torch.__version__) > version.parse("1.6.0"):
128
+ self.register_buffer(
129
+ "token_type_ids",
130
+ torch.zeros(self.position_ids.size(), dtype=torch.long),
131
+ persistent=False,
132
+ )
133
+
134
+ # End copy
135
+ self.padding_idx = config.pad_token_id
136
+ self.position_embeddings = nn.Embedding(
137
+ config.max_position_embeddings,
138
+ config.hidden_size,
139
+ padding_idx=self.padding_idx,
140
+ )
141
+
142
+ def forward(
143
+ self,
144
+ input_ids=None,
145
+ token_type_ids=None,
146
+ position_ids=None,
147
+ inputs_embeds=None,
148
+ past_key_values_length=0,
149
+ ):
150
+ if position_ids is None:
151
+ if input_ids is not None:
152
+ # Create the position ids from the input token ids. Any padded tokens remain padded.
153
+ position_ids = create_position_ids_from_input_ids(
154
+ input_ids, self.padding_idx, past_key_values_length
155
+ )
156
+ else:
157
+ position_ids = self.create_position_ids_from_inputs_embeds(
158
+ inputs_embeds
159
+ )
160
+
161
+ if input_ids is not None:
162
+ input_shape = input_ids.size()
163
+ else:
164
+ input_shape = inputs_embeds.size()[:-1]
165
+
166
+ seq_length = input_shape[1]
167
+
168
+ # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs
169
+ # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
170
+ # issue #5664
171
+ if token_type_ids is None:
172
+ if hasattr(self, "token_type_ids"):
173
+ buffered_token_type_ids = self.token_type_ids[:, :seq_length]
174
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
175
+ input_shape[0], seq_length
176
+ )
177
+ token_type_ids = buffered_token_type_ids_expanded
178
+ else:
179
+ token_type_ids = torch.zeros(
180
+ input_shape, dtype=torch.long, device=self.position_ids.device
181
+ )
182
+
183
+ if inputs_embeds is None:
184
+ inputs_embeds = self.word_embeddings(input_ids)
185
+ token_type_embeddings = self.token_type_embeddings(token_type_ids)
186
+
187
+ embeddings = inputs_embeds + token_type_embeddings
188
+ if self.position_embedding_type == "absolute":
189
+ position_embeddings = self.position_embeddings(position_ids)
190
+ embeddings += position_embeddings
191
+ embeddings = self.LayerNorm(embeddings)
192
+ embeddings = self.dropout(embeddings)
193
+ return embeddings
194
+
195
+ def create_position_ids_from_inputs_embeds(self, inputs_embeds):
196
+ """
197
+ We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.
198
+
199
+ Args:
200
+ inputs_embeds: torch.Tensor
201
+
202
+ Returns: torch.Tensor
203
+ """
204
+ input_shape = inputs_embeds.size()[:-1]
205
+ sequence_length = input_shape[1]
206
+
207
+ position_ids = torch.arange(
208
+ self.padding_idx + 1,
209
+ sequence_length + self.padding_idx + 1,
210
+ dtype=torch.long,
211
+ device=inputs_embeds.device,
212
+ )
213
+ return position_ids.unsqueeze(0).expand(input_shape)
214
+
215
+
216
+ # Copied from transformers.models.bert.modeling_bert.BertSelfAttention with Bert->Roberta
217
+ class RobertaSelfAttention(nn.Module):
218
+ def __init__(self, config, position_embedding_type=None):
219
+ super().__init__()
220
+ if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
221
+ config, "embedding_size"
222
+ ):
223
+ raise ValueError(
224
+ f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
225
+ f"heads ({config.num_attention_heads})"
226
+ )
227
+
228
+ self.num_attention_heads = config.num_attention_heads
229
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
230
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
231
+
232
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
233
+ self.key = nn.Linear(config.hidden_size, self.all_head_size)
234
+ self.value = nn.Linear(config.hidden_size, self.all_head_size)
235
+
236
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
237
+ self.position_embedding_type = position_embedding_type or getattr(
238
+ config, "position_embedding_type", "absolute"
239
+ )
240
+ if (
241
+ self.position_embedding_type == "relative_key"
242
+ or self.position_embedding_type == "relative_key_query"
243
+ ):
244
+ self.max_position_embeddings = config.max_position_embeddings
245
+ self.distance_embedding = nn.Embedding(
246
+ 2 * config.max_position_embeddings - 1, self.attention_head_size
247
+ )
248
+
249
+ self.is_decoder = config.is_decoder
250
+
251
+ def transpose_for_scores(self, x):
252
+ new_x_shape = x.size()[:-1] + (
253
+ self.num_attention_heads,
254
+ self.attention_head_size,
255
+ )
256
+ x = x.view(new_x_shape)
257
+ return x.permute(0, 2, 1, 3)
258
+
259
+ def forward(
260
+ self,
261
+ hidden_states: torch.Tensor,
262
+ attention_mask: Optional[torch.FloatTensor] = None,
263
+ head_mask: Optional[torch.FloatTensor] = None,
264
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
265
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
266
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
267
+ output_attentions: Optional[bool] = False,
268
+ parser_att_mask=None,
269
+ ) -> Tuple[torch.Tensor]:
270
+ mixed_query_layer = self.query(hidden_states)
271
+
272
+ # If this is instantiated as a cross-attention module, the keys
273
+ # and values come from an encoder; the attention mask needs to be
274
+ # such that the encoder's padding tokens are not attended to.
275
+ is_cross_attention = encoder_hidden_states is not None
276
+
277
+ if is_cross_attention and past_key_value is not None:
278
+ # reuse k,v, cross_attentions
279
+ key_layer = past_key_value[0]
280
+ value_layer = past_key_value[1]
281
+ attention_mask = encoder_attention_mask
282
+ elif is_cross_attention:
283
+ key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
284
+ value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
285
+ attention_mask = encoder_attention_mask
286
+ elif past_key_value is not None:
287
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
288
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
289
+ key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
290
+ value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
291
+ else:
292
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
293
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
294
+
295
+ query_layer = self.transpose_for_scores(mixed_query_layer)
296
+
297
+ if self.is_decoder:
298
+ # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
299
+ # Further calls to cross_attention layer can then reuse all cross-attention
300
+ # key/value_states (first "if" case)
301
+ # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
302
+ # all previous decoder key/value_states. Further calls to uni-directional self-attention
303
+ # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
304
+ # if encoder bi-directional self-attention `past_key_value` is always `None`
305
+ past_key_value = (key_layer, value_layer)
306
+
307
+ # Take the dot product between "query" and "key" to get the raw attention scores.
308
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
309
+
310
+ if (
311
+ self.position_embedding_type == "relative_key"
312
+ or self.position_embedding_type == "relative_key_query"
313
+ ):
314
+ seq_length = hidden_states.size()[1]
315
+ position_ids_l = torch.arange(
316
+ seq_length, dtype=torch.long, device=hidden_states.device
317
+ ).view(-1, 1)
318
+ position_ids_r = torch.arange(
319
+ seq_length, dtype=torch.long, device=hidden_states.device
320
+ ).view(1, -1)
321
+ distance = position_ids_l - position_ids_r
322
+ positional_embedding = self.distance_embedding(
323
+ distance + self.max_position_embeddings - 1
324
+ )
325
+ positional_embedding = positional_embedding.to(
326
+ dtype=query_layer.dtype
327
+ ) # fp16 compatibility
328
+
329
+ if self.position_embedding_type == "relative_key":
330
+ relative_position_scores = torch.einsum(
331
+ "bhld,lrd->bhlr", query_layer, positional_embedding
332
+ )
333
+ attention_scores = attention_scores + relative_position_scores
334
+ elif self.position_embedding_type == "relative_key_query":
335
+ relative_position_scores_query = torch.einsum(
336
+ "bhld,lrd->bhlr", query_layer, positional_embedding
337
+ )
338
+ relative_position_scores_key = torch.einsum(
339
+ "bhrd,lrd->bhlr", key_layer, positional_embedding
340
+ )
341
+ attention_scores = (
342
+ attention_scores
343
+ + relative_position_scores_query
344
+ + relative_position_scores_key
345
+ )
346
+
347
+ attention_scores = attention_scores / math.sqrt(self.attention_head_size)
348
+ if attention_mask is not None:
349
+ # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
350
+ attention_scores = attention_scores + attention_mask
351
+
352
+ if parser_att_mask is None:
353
+ # Normalize the attention scores to probabilities.
354
+ attention_probs = nn.functional.softmax(attention_scores, dim=-1)
355
+ else:
356
+ attention_probs = torch.sigmoid(attention_scores) * parser_att_mask
357
+
358
+ # This is actually dropping out entire tokens to attend to, which might
359
+ # seem a bit unusual, but is taken from the original Transformer paper.
360
+ attention_probs = self.dropout(attention_probs)
361
+
362
+ # Mask heads if we want to
363
+ if head_mask is not None:
364
+ attention_probs = attention_probs * head_mask
365
+
366
+ context_layer = torch.matmul(attention_probs, value_layer)
367
+
368
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
369
+ new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
370
+ context_layer = context_layer.view(new_context_layer_shape)
371
+
372
+ outputs = (
373
+ (context_layer, attention_probs) if output_attentions else (context_layer,)
374
+ )
375
+
376
+ if self.is_decoder:
377
+ outputs = outputs + (past_key_value,)
378
+ return outputs
379
+
380
+
381
+ # Copied from transformers.models.bert.modeling_bert.BertSelfOutput
382
+ class RobertaSelfOutput(nn.Module):
383
+ def __init__(self, config):
384
+ super().__init__()
385
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
386
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
387
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
388
+
389
+ def forward(
390
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
391
+ ) -> torch.Tensor:
392
+ hidden_states = self.dense(hidden_states)
393
+ hidden_states = self.dropout(hidden_states)
394
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
395
+ return hidden_states
396
+
397
+
398
+ # Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Roberta
399
+ class RobertaAttention(nn.Module):
400
+ def __init__(self, config, position_embedding_type=None):
401
+ super().__init__()
402
+ self.self = RobertaSelfAttention(
403
+ config, position_embedding_type=position_embedding_type
404
+ )
405
+ self.output = RobertaSelfOutput(config)
406
+ self.pruned_heads = set()
407
+
408
+ def prune_heads(self, heads):
409
+ if len(heads) == 0:
410
+ return
411
+ heads, index = find_pruneable_heads_and_indices(
412
+ heads,
413
+ self.self.num_attention_heads,
414
+ self.self.attention_head_size,
415
+ self.pruned_heads,
416
+ )
417
+
418
+ # Prune linear layers
419
+ self.self.query = prune_linear_layer(self.self.query, index)
420
+ self.self.key = prune_linear_layer(self.self.key, index)
421
+ self.self.value = prune_linear_layer(self.self.value, index)
422
+ self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
423
+
424
+ # Update hyper params and store pruned heads
425
+ self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
426
+ self.self.all_head_size = (
427
+ self.self.attention_head_size * self.self.num_attention_heads
428
+ )
429
+ self.pruned_heads = self.pruned_heads.union(heads)
430
+
431
+ def forward(
432
+ self,
433
+ hidden_states: torch.Tensor,
434
+ attention_mask: Optional[torch.FloatTensor] = None,
435
+ head_mask: Optional[torch.FloatTensor] = None,
436
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
437
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
438
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
439
+ output_attentions: Optional[bool] = False,
440
+ parser_att_mask=None,
441
+ ) -> Tuple[torch.Tensor]:
442
+ self_outputs = self.self(
443
+ hidden_states,
444
+ attention_mask,
445
+ head_mask,
446
+ encoder_hidden_states,
447
+ encoder_attention_mask,
448
+ past_key_value,
449
+ output_attentions,
450
+ parser_att_mask=parser_att_mask,
451
+ )
452
+ attention_output = self.output(self_outputs[0], hidden_states)
453
+ outputs = (attention_output,) + self_outputs[
454
+ 1:
455
+ ] # add attentions if we output them
456
+ return outputs
457
+
458
+
459
+ # Copied from transformers.models.bert.modeling_bert.BertIntermediate
460
+ class RobertaIntermediate(nn.Module):
461
+ def __init__(self, config):
462
+ super().__init__()
463
+ self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
464
+ if isinstance(config.hidden_act, str):
465
+ self.intermediate_act_fn = ACT2FN[config.hidden_act]
466
+ else:
467
+ self.intermediate_act_fn = config.hidden_act
468
+
469
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
470
+ hidden_states = self.dense(hidden_states)
471
+ hidden_states = self.intermediate_act_fn(hidden_states)
472
+ return hidden_states
473
+
474
+
475
+ # Copied from transformers.models.bert.modeling_bert.BertOutput
476
+ class RobertaOutput(nn.Module):
477
+ def __init__(self, config):
478
+ super().__init__()
479
+ self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
480
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
481
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
482
+
483
+ def forward(
484
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
485
+ ) -> torch.Tensor:
486
+ hidden_states = self.dense(hidden_states)
487
+ hidden_states = self.dropout(hidden_states)
488
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
489
+ return hidden_states
490
+
491
+
492
+ # Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Roberta
493
+ class RobertaLayer(nn.Module):
494
+ def __init__(self, config):
495
+ super().__init__()
496
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
497
+ self.seq_len_dim = 1
498
+ self.attention = RobertaAttention(config)
499
+ self.is_decoder = config.is_decoder
500
+ self.add_cross_attention = config.add_cross_attention
501
+ if self.add_cross_attention:
502
+ if not self.is_decoder:
503
+ raise ValueError(
504
+ f"{self} should be used as a decoder model if cross attention is added"
505
+ )
506
+ self.crossattention = RobertaAttention(
507
+ config, position_embedding_type="absolute"
508
+ )
509
+ self.intermediate = RobertaIntermediate(config)
510
+ self.output = RobertaOutput(config)
511
+
512
+ def forward(
513
+ self,
514
+ hidden_states: torch.Tensor,
515
+ attention_mask: Optional[torch.FloatTensor] = None,
516
+ head_mask: Optional[torch.FloatTensor] = None,
517
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
518
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
519
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
520
+ output_attentions: Optional[bool] = False,
521
+ parser_att_mask=None,
522
+ ) -> Tuple[torch.Tensor]:
523
+ # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
524
+ self_attn_past_key_value = (
525
+ past_key_value[:2] if past_key_value is not None else None
526
+ )
527
+ self_attention_outputs = self.attention(
528
+ hidden_states,
529
+ attention_mask,
530
+ head_mask,
531
+ output_attentions=output_attentions,
532
+ past_key_value=self_attn_past_key_value,
533
+ parser_att_mask=parser_att_mask,
534
+ )
535
+ attention_output = self_attention_outputs[0]
536
+
537
+ # if decoder, the last output is tuple of self-attn cache
538
+ if self.is_decoder:
539
+ outputs = self_attention_outputs[1:-1]
540
+ present_key_value = self_attention_outputs[-1]
541
+ else:
542
+ outputs = self_attention_outputs[
543
+ 1:
544
+ ] # add self attentions if we output attention weights
545
+
546
+ cross_attn_present_key_value = None
547
+ if self.is_decoder and encoder_hidden_states is not None:
548
+ if not hasattr(self, "crossattention"):
549
+ raise ValueError(
550
+ f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`"
551
+ )
552
+
553
+ # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
554
+ cross_attn_past_key_value = (
555
+ past_key_value[-2:] if past_key_value is not None else None
556
+ )
557
+ cross_attention_outputs = self.crossattention(
558
+ attention_output,
559
+ attention_mask,
560
+ head_mask,
561
+ encoder_hidden_states,
562
+ encoder_attention_mask,
563
+ cross_attn_past_key_value,
564
+ output_attentions,
565
+ )
566
+ attention_output = cross_attention_outputs[0]
567
+ outputs = (
568
+ outputs + cross_attention_outputs[1:-1]
569
+ ) # add cross attentions if we output attention weights
570
+
571
+ # add cross-attn cache to positions 3,4 of present_key_value tuple
572
+ cross_attn_present_key_value = cross_attention_outputs[-1]
573
+ present_key_value = present_key_value + cross_attn_present_key_value
574
+
575
+ layer_output = apply_chunking_to_forward(
576
+ self.feed_forward_chunk,
577
+ self.chunk_size_feed_forward,
578
+ self.seq_len_dim,
579
+ attention_output,
580
+ )
581
+ outputs = (layer_output,) + outputs
582
+
583
+ # if decoder, return the attn key/values as the last output
584
+ if self.is_decoder:
585
+ outputs = outputs + (present_key_value,)
586
+
587
+ return outputs
588
+
589
+ def feed_forward_chunk(self, attention_output):
590
+ intermediate_output = self.intermediate(attention_output)
591
+ layer_output = self.output(intermediate_output, attention_output)
592
+ return layer_output
593
+
594
+
595
+ # Copied from transformers.models.bert.modeling_bert.BertEncoder with Bert->Roberta
596
+ class RobertaEncoder(nn.Module):
597
+ def __init__(self, config):
598
+ super().__init__()
599
+ self.config = config
600
+ self.layer = nn.ModuleList(
601
+ [RobertaLayer(config) for _ in range(config.num_hidden_layers)]
602
+ )
603
+ self.gradient_checkpointing = False
604
+
605
+ def forward(
606
+ self,
607
+ hidden_states: torch.Tensor,
608
+ attention_mask: Optional[torch.FloatTensor] = None,
609
+ head_mask: Optional[torch.FloatTensor] = None,
610
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
611
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
612
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
613
+ use_cache: Optional[bool] = None,
614
+ output_attentions: Optional[bool] = False,
615
+ output_hidden_states: Optional[bool] = False,
616
+ return_dict: Optional[bool] = True,
617
+ parser_att_mask=None,
618
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]:
619
+ all_hidden_states = () if output_hidden_states else None
620
+ all_self_attentions = () if output_attentions else None
621
+ all_cross_attentions = (
622
+ () if output_attentions and self.config.add_cross_attention else None
623
+ )
624
+
625
+ next_decoder_cache = () if use_cache else None
626
+ for i, layer_module in enumerate(self.layer):
627
+ if output_hidden_states:
628
+ all_hidden_states = all_hidden_states + (hidden_states,)
629
+
630
+ layer_head_mask = head_mask[i] if head_mask is not None else None
631
+ past_key_value = past_key_values[i] if past_key_values is not None else None
632
+
633
+ if self.gradient_checkpointing and self.training:
634
+
635
+ if use_cache:
636
+ logger.warning(
637
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
638
+ )
639
+ use_cache = False
640
+
641
+ def create_custom_forward(module):
642
+ def custom_forward(*inputs):
643
+ return module(*inputs, past_key_value, output_attentions)
644
+
645
+ return custom_forward
646
+
647
+ layer_outputs = torch.utils.checkpoint.checkpoint(
648
+ create_custom_forward(layer_module),
649
+ hidden_states,
650
+ attention_mask,
651
+ layer_head_mask,
652
+ encoder_hidden_states,
653
+ encoder_attention_mask,
654
+ )
655
+ else:
656
+ if parser_att_mask is not None:
657
+ layer_outputs = layer_module(
658
+ hidden_states,
659
+ attention_mask,
660
+ layer_head_mask,
661
+ encoder_hidden_states,
662
+ encoder_attention_mask,
663
+ past_key_value,
664
+ output_attentions,
665
+ parser_att_mask=parser_att_mask[i])
666
+ else:
667
+ layer_outputs = layer_module(
668
+ hidden_states,
669
+ attention_mask,
670
+ layer_head_mask,
671
+ encoder_hidden_states,
672
+ encoder_attention_mask,
673
+ past_key_value,
674
+ output_attentions,
675
+ parser_att_mask=None)
676
+
677
+
678
+ hidden_states = layer_outputs[0]
679
+ if use_cache:
680
+ next_decoder_cache += (layer_outputs[-1],)
681
+ if output_attentions:
682
+ all_self_attentions = all_self_attentions + (layer_outputs[1],)
683
+ if self.config.add_cross_attention:
684
+ all_cross_attentions = all_cross_attentions + (layer_outputs[2],)
685
+
686
+ if output_hidden_states:
687
+ all_hidden_states = all_hidden_states + (hidden_states,)
688
+
689
+ if not return_dict:
690
+ return tuple(
691
+ v
692
+ for v in [
693
+ hidden_states,
694
+ next_decoder_cache,
695
+ all_hidden_states,
696
+ all_self_attentions,
697
+ all_cross_attentions,
698
+ ]
699
+ if v is not None
700
+ )
701
+ return BaseModelOutputWithPastAndCrossAttentions(
702
+ last_hidden_state=hidden_states,
703
+ past_key_values=next_decoder_cache,
704
+ hidden_states=all_hidden_states,
705
+ attentions=all_self_attentions,
706
+ cross_attentions=all_cross_attentions,
707
+ )
708
+
709
+
710
+ # Copied from transformers.models.bert.modeling_bert.BertPooler
711
+ class RobertaPooler(nn.Module):
712
+ def __init__(self, config):
713
+ super().__init__()
714
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
715
+ self.activation = nn.Tanh()
716
+
717
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
718
+ # We "pool" the model by simply taking the hidden state corresponding
719
+ # to the first token.
720
+ first_token_tensor = hidden_states[:, 0]
721
+ pooled_output = self.dense(first_token_tensor)
722
+ pooled_output = self.activation(pooled_output)
723
+ return pooled_output
724
+
725
+
726
+ class RobertaPreTrainedModel(PreTrainedModel):
727
+ """
728
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
729
+ models.
730
+ """
731
+
732
+ config_class = RobertaConfig
733
+ base_model_prefix = "roberta"
734
+ supports_gradient_checkpointing = True
735
+
736
+ # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights
737
+ def _init_weights(self, module):
738
+ """Initialize the weights"""
739
+ if isinstance(module, nn.Linear):
740
+ # Slightly different from the TF version which uses truncated_normal for initialization
741
+ # cf https://github.com/pytorch/pytorch/pull/5617
742
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
743
+ if module.bias is not None:
744
+ module.bias.data.zero_()
745
+ elif isinstance(module, nn.Embedding):
746
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
747
+ if module.padding_idx is not None:
748
+ module.weight.data[module.padding_idx].zero_()
749
+ elif isinstance(module, nn.LayerNorm):
750
+ if module.bias is not None:
751
+ module.bias.data.zero_()
752
+ module.weight.data.fill_(1.0)
753
+
754
+ def _set_gradient_checkpointing(self, module, value=False):
755
+ if isinstance(module, RobertaEncoder):
756
+ module.gradient_checkpointing = value
757
+
758
+ def update_keys_to_ignore(self, config, del_keys_to_ignore):
759
+ """Remove some keys from ignore list"""
760
+ if not config.tie_word_embeddings:
761
+ # must make a new list, or the class variable gets modified!
762
+ self._keys_to_ignore_on_save = [
763
+ k for k in self._keys_to_ignore_on_save if k not in del_keys_to_ignore
764
+ ]
765
+ self._keys_to_ignore_on_load_missing = [
766
+ k
767
+ for k in self._keys_to_ignore_on_load_missing
768
+ if k not in del_keys_to_ignore
769
+ ]
770
+
771
+
772
+ ROBERTA_START_DOCSTRING = r"""
773
+
774
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
775
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
776
+ etc.)
777
+
778
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
779
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
780
+ and behavior.
781
+
782
+ Parameters:
783
+ config ([`RobertaConfig`]): Model configuration class with all the parameters of the
784
+ model. Initializing with a config file does not load the weights associated with the model, only the
785
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
786
+ """
787
+
788
+
789
+ ROBERTA_INPUTS_DOCSTRING = r"""
790
+ Args:
791
+ input_ids (`torch.LongTensor` of shape `({0})`):
792
+ Indices of input sequence tokens in the vocabulary.
793
+
794
+ Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
795
+ [`PreTrainedTokenizer.__call__`] for details.
796
+
797
+ [What are input IDs?](../glossary#input-ids)
798
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
799
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
800
+
801
+ - 1 for tokens that are **not masked**,
802
+ - 0 for tokens that are **masked**.
803
+
804
+ [What are attention masks?](../glossary#attention-mask)
805
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
806
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
807
+ 1]`:
808
+
809
+ - 0 corresponds to a *sentence A* token,
810
+ - 1 corresponds to a *sentence B* token.
811
+
812
+ [What are token type IDs?](../glossary#token-type-ids)
813
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
814
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
815
+ config.max_position_embeddings - 1]`.
816
+
817
+ [What are position IDs?](../glossary#position-ids)
818
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
819
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
820
+
821
+ - 1 indicates the head is **not masked**,
822
+ - 0 indicates the head is **masked**.
823
+
824
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
825
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
826
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
827
+ model's internal embedding lookup matrix.
828
+ output_attentions (`bool`, *optional*):
829
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
830
+ tensors for more detail.
831
+ output_hidden_states (`bool`, *optional*):
832
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
833
+ more detail.
834
+ return_dict (`bool`, *optional*):
835
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
836
+ """
837
+
838
+
839
+ class RobertaModel(RobertaPreTrainedModel):
840
+ """
841
+
842
+ The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
843
+ cross-attention is added between the self-attention layers, following the architecture described in *Attention is
844
+ all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
845
+ Kaiser and Illia Polosukhin.
846
+
847
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
848
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
849
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
850
+
851
+ .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762
852
+
853
+ """
854
+
855
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
856
+
857
+ # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->Roberta
858
+ def __init__(self, config, add_pooling_layer=True):
859
+ super().__init__(config)
860
+ self.config = config
861
+
862
+ self.embeddings = RobertaEmbeddings(config)
863
+ self.encoder = RobertaEncoder(config)
864
+
865
+ self.pooler = RobertaPooler(config) if add_pooling_layer else None
866
+
867
+ # Initialize weights and apply final processing
868
+ self.post_init()
869
+
870
+ def get_input_embeddings(self):
871
+ return self.embeddings.word_embeddings
872
+
873
+ def set_input_embeddings(self, value):
874
+ self.embeddings.word_embeddings = value
875
+
876
+ def _prune_heads(self, heads_to_prune):
877
+ """
878
+ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
879
+ class PreTrainedModel
880
+ """
881
+ for layer, heads in heads_to_prune.items():
882
+ self.encoder.layer[layer].attention.prune_heads(heads)
883
+
884
+ # Copied from transformers.models.bert.modeling_bert.BertModel.forward
885
+ def forward(
886
+ self,
887
+ input_ids: Optional[torch.Tensor] = None,
888
+ attention_mask: Optional[torch.Tensor] = None,
889
+ token_type_ids: Optional[torch.Tensor] = None,
890
+ position_ids: Optional[torch.Tensor] = None,
891
+ head_mask: Optional[torch.Tensor] = None,
892
+ inputs_embeds: Optional[torch.Tensor] = None,
893
+ encoder_hidden_states: Optional[torch.Tensor] = None,
894
+ encoder_attention_mask: Optional[torch.Tensor] = None,
895
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
896
+ use_cache: Optional[bool] = None,
897
+ output_attentions: Optional[bool] = None,
898
+ output_hidden_states: Optional[bool] = None,
899
+ return_dict: Optional[bool] = None,
900
+ parser_att_mask=None,
901
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]:
902
+ r"""
903
+ encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
904
+ Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
905
+ the model is configured as a decoder.
906
+ encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
907
+ Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
908
+ the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:
909
+
910
+ - 1 for tokens that are **not masked**,
911
+ - 0 for tokens that are **masked**.
912
+ past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
913
+ Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
914
+
915
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
916
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
917
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
918
+ use_cache (`bool`, *optional*):
919
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
920
+ `past_key_values`).
921
+ """
922
+ output_attentions = (
923
+ output_attentions
924
+ if output_attentions is not None
925
+ else self.config.output_attentions
926
+ )
927
+ output_hidden_states = (
928
+ output_hidden_states
929
+ if output_hidden_states is not None
930
+ else self.config.output_hidden_states
931
+ )
932
+ return_dict = (
933
+ return_dict if return_dict is not None else self.config.use_return_dict
934
+ )
935
+
936
+ if self.config.is_decoder:
937
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
938
+ else:
939
+ use_cache = False
940
+
941
+ if input_ids is not None and inputs_embeds is not None:
942
+ raise ValueError(
943
+ "You cannot specify both input_ids and inputs_embeds at the same time"
944
+ )
945
+ elif input_ids is not None:
946
+ input_shape = input_ids.size()
947
+ elif inputs_embeds is not None:
948
+ input_shape = inputs_embeds.size()[:-1]
949
+ else:
950
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
951
+
952
+ batch_size, seq_length = input_shape
953
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
954
+
955
+ # past_key_values_length
956
+ past_key_values_length = (
957
+ past_key_values[0][0].shape[2] if past_key_values is not None else 0
958
+ )
959
+
960
+ if attention_mask is None:
961
+ attention_mask = torch.ones(
962
+ ((batch_size, seq_length + past_key_values_length)), device=device
963
+ )
964
+
965
+ if token_type_ids is None:
966
+ if hasattr(self.embeddings, "token_type_ids"):
967
+ buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
968
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
969
+ batch_size, seq_length
970
+ )
971
+ token_type_ids = buffered_token_type_ids_expanded
972
+ else:
973
+ token_type_ids = torch.zeros(
974
+ input_shape, dtype=torch.long, device=device
975
+ )
976
+
977
+ # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
978
+ # ourselves in which case we just need to make it broadcastable to all heads.
979
+ extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(
980
+ attention_mask, input_shape, device
981
+ )
982
+
983
+ # If a 2D or 3D attention mask is provided for the cross-attention
984
+ # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
985
+ if self.config.is_decoder and encoder_hidden_states is not None:
986
+ (
987
+ encoder_batch_size,
988
+ encoder_sequence_length,
989
+ _,
990
+ ) = encoder_hidden_states.size()
991
+ encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
992
+ if encoder_attention_mask is None:
993
+ encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
994
+ encoder_extended_attention_mask = self.invert_attention_mask(
995
+ encoder_attention_mask
996
+ )
997
+ else:
998
+ encoder_extended_attention_mask = None
999
+
1000
+ # Prepare head mask if needed
1001
+ # 1.0 in head_mask indicate we keep the head
1002
+ # attention_probs has shape bsz x n_heads x N x N
1003
+ # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
1004
+ # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
1005
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
1006
+
1007
+ embedding_output = self.embeddings(
1008
+ input_ids=input_ids,
1009
+ position_ids=position_ids,
1010
+ token_type_ids=token_type_ids,
1011
+ inputs_embeds=inputs_embeds,
1012
+ past_key_values_length=past_key_values_length,
1013
+ )
1014
+ encoder_outputs = self.encoder(
1015
+ embedding_output,
1016
+ attention_mask=extended_attention_mask,
1017
+ head_mask=head_mask,
1018
+ encoder_hidden_states=encoder_hidden_states,
1019
+ encoder_attention_mask=encoder_extended_attention_mask,
1020
+ past_key_values=past_key_values,
1021
+ use_cache=use_cache,
1022
+ output_attentions=output_attentions,
1023
+ output_hidden_states=output_hidden_states,
1024
+ return_dict=return_dict,
1025
+ parser_att_mask=parser_att_mask,
1026
+ )
1027
+ sequence_output = encoder_outputs[0]
1028
+ pooled_output = (
1029
+ self.pooler(sequence_output) if self.pooler is not None else None
1030
+ )
1031
+
1032
+ if not return_dict:
1033
+ return (sequence_output, pooled_output) + encoder_outputs[1:]
1034
+
1035
+ return BaseModelOutputWithPoolingAndCrossAttentions(
1036
+ last_hidden_state=sequence_output,
1037
+ pooler_output=pooled_output,
1038
+ past_key_values=encoder_outputs.past_key_values,
1039
+ hidden_states=encoder_outputs.hidden_states,
1040
+ attentions=encoder_outputs.attentions,
1041
+ cross_attentions=encoder_outputs.cross_attentions,
1042
+ )
1043
+
1044
+
1045
+ class StructRoberta(RobertaPreTrainedModel):
1046
+ _keys_to_ignore_on_save = [r"lm_head.decoder.weight", r"lm_head.decoder.bias"]
1047
+ _keys_to_ignore_on_load_missing = [
1048
+ r"position_ids",
1049
+ r"lm_head.decoder.weight",
1050
+ r"lm_head.decoder.bias",
1051
+ ]
1052
+ _keys_to_ignore_on_load_unexpected = [r"pooler"]
1053
+
1054
+ def __init__(self, config):
1055
+ super().__init__(config)
1056
+
1057
+ if config.is_decoder:
1058
+ logger.warning(
1059
+ "If you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for "
1060
+ "bi-directional self-attention."
1061
+ )
1062
+
1063
+
1064
+ if config.n_cntxt_layers > 0:
1065
+ config_cntxt = copy.deepcopy(config)
1066
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1067
+
1068
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1069
+
1070
+ if config.n_cntxt_layers_2 > 0:
1071
+ self.parser_layers_1 = nn.ModuleList(
1072
+ [
1073
+ nn.Sequential(
1074
+ Conv1d(config.hidden_size, config.conv_size),
1075
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1076
+ nn.Tanh(),
1077
+ )
1078
+ for i in range(int(config.n_parser_layers/2))
1079
+ ]
1080
+ )
1081
+
1082
+ self.distance_ff_1 = nn.Sequential(
1083
+ Conv1d(config.hidden_size, 2),
1084
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1085
+ nn.Tanh(),
1086
+ nn.Linear(config.hidden_size, 1),
1087
+ )
1088
+
1089
+ self.height_ff_1 = nn.Sequential(
1090
+ nn.Linear(config.hidden_size, config.hidden_size),
1091
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1092
+ nn.Tanh(),
1093
+ nn.Linear(config.hidden_size, 1),
1094
+ )
1095
+
1096
+ n_rel = len(config.relations)
1097
+ self._rel_weight_1 = nn.Parameter(
1098
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1099
+ )
1100
+ self._rel_weight_1.data.normal_(0, 0.1)
1101
+
1102
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1103
+
1104
+ config_cntxt_2 = copy.deepcopy(config)
1105
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1106
+
1107
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1108
+
1109
+
1110
+ self.parser_layers_2 = nn.ModuleList(
1111
+ [
1112
+ nn.Sequential(
1113
+ Conv1d(config.hidden_size, config.conv_size),
1114
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1115
+ nn.Tanh(),
1116
+ )
1117
+ for i in range(int(config.n_parser_layers/2))
1118
+ ]
1119
+ )
1120
+
1121
+ self.distance_ff_2 = nn.Sequential(
1122
+ Conv1d(config.hidden_size, 2),
1123
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1124
+ nn.Tanh(),
1125
+ nn.Linear(config.hidden_size, 1),
1126
+ )
1127
+
1128
+ self.height_ff_2 = nn.Sequential(
1129
+ nn.Linear(config.hidden_size, config.hidden_size),
1130
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1131
+ nn.Tanh(),
1132
+ nn.Linear(config.hidden_size, 1),
1133
+ )
1134
+
1135
+ n_rel = len(config.relations)
1136
+ self._rel_weight_2 = nn.Parameter(
1137
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1138
+ )
1139
+ self._rel_weight_2.data.normal_(0, 0.1)
1140
+
1141
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1142
+
1143
+ else:
1144
+ self.parser_layers = nn.ModuleList(
1145
+ [
1146
+ nn.Sequential(
1147
+ Conv1d(config.hidden_size, config.conv_size),
1148
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1149
+ nn.Tanh(),
1150
+ )
1151
+ for i in range(config.n_parser_layers)
1152
+ ]
1153
+ )
1154
+
1155
+ self.distance_ff = nn.Sequential(
1156
+ Conv1d(config.hidden_size, 2),
1157
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1158
+ nn.Tanh(),
1159
+ nn.Linear(config.hidden_size, 1),
1160
+ )
1161
+
1162
+ self.height_ff = nn.Sequential(
1163
+ nn.Linear(config.hidden_size, config.hidden_size),
1164
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1165
+ nn.Tanh(),
1166
+ nn.Linear(config.hidden_size, 1),
1167
+ )
1168
+
1169
+ n_rel = len(config.relations)
1170
+ self._rel_weight = nn.Parameter(
1171
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1172
+ )
1173
+ self._rel_weight.data.normal_(0, 0.1)
1174
+
1175
+ self._scaler = nn.Parameter(torch.zeros(2))
1176
+
1177
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1178
+
1179
+ if config.n_cntxt_layers > 0:
1180
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1181
+ if config.n_cntxt_layers_2 > 0:
1182
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1183
+
1184
+ self.lm_head = RobertaLMHead(config)
1185
+
1186
+ self.pad = config.pad_token_id
1187
+
1188
+ # The LM head weights require special treatment only when they are tied with the word embeddings
1189
+ self.update_keys_to_ignore(config, ["lm_head.decoder.weight"])
1190
+
1191
+ # Initialize weights and apply final processing
1192
+ self.post_init()
1193
+
1194
+ def get_output_embeddings(self):
1195
+ return self.lm_head.decoder
1196
+
1197
+ def set_output_embeddings(self, new_embeddings):
1198
+ self.lm_head.decoder = new_embeddings
1199
+
1200
+ @property
1201
+ def scaler(self):
1202
+ return self._scaler.exp()
1203
+
1204
+ @property
1205
+ def scaler_1(self):
1206
+ return self._scaler_1.exp()
1207
+
1208
+ @property
1209
+ def scaler_2(self):
1210
+ return self._scaler_2.exp()
1211
+
1212
+ @property
1213
+ def rel_weight(self):
1214
+ if self.config.weight_act == "sigmoid":
1215
+ return torch.sigmoid(self._rel_weight)
1216
+ elif self.config.weight_act == "softmax":
1217
+ return torch.softmax(self._rel_weight, dim=-1)
1218
+
1219
+ @property
1220
+ def rel_weight_1(self):
1221
+ if self.config.weight_act == "sigmoid":
1222
+ return torch.sigmoid(self._rel_weight_1)
1223
+ elif self.config.weight_act == "softmax":
1224
+ return torch.softmax(self._rel_weight_1, dim=-1)
1225
+
1226
+
1227
+ @property
1228
+ def rel_weight_2(self):
1229
+ if self.config.weight_act == "sigmoid":
1230
+ return torch.sigmoid(self._rel_weight_2)
1231
+ elif self.config.weight_act == "softmax":
1232
+ return torch.softmax(self._rel_weight_2, dim=-1)
1233
+
1234
+
1235
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1236
+ """Compute constituents from distance and height."""
1237
+
1238
+ if n_cntxt_layers>0:
1239
+ if n_cntxt_layers == 1:
1240
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1241
+ elif n_cntxt_layers == 2:
1242
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1243
+ else:
1244
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1245
+
1246
+ gamma = torch.sigmoid(-beta_logits)
1247
+ ones = torch.ones_like(gamma)
1248
+
1249
+ block_mask_left = cummin(
1250
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1251
+ )
1252
+ block_mask_left = block_mask_left - F.pad(
1253
+ block_mask_left[:, :, :-1], (1, 0), value=0
1254
+ )
1255
+ block_mask_left.tril_(0)
1256
+
1257
+ block_mask_right = cummin(
1258
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1259
+ )
1260
+ block_mask_right = block_mask_right - F.pad(
1261
+ block_mask_right[:, :, 1:], (0, 1), value=0
1262
+ )
1263
+ block_mask_right.triu_(0)
1264
+
1265
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1266
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1267
+ block_mask_right, reverse=True
1268
+ ).triu(1)
1269
+
1270
+ return block_p, block
1271
+
1272
+ def compute_head(self, height, n_cntxt_layers=0):
1273
+ """Estimate head for each constituent."""
1274
+
1275
+ _, length = height.size()
1276
+ if n_cntxt_layers>0:
1277
+ if n_cntxt_layers == 1:
1278
+ head_logits = height * self.scaler_1[1]
1279
+ elif n_cntxt_layers == 2:
1280
+ head_logits = height * self.scaler_2[1]
1281
+ else:
1282
+ head_logits = height * self.scaler[1]
1283
+ index = torch.arange(length, device=height.device)
1284
+
1285
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1286
+ index[None, None, :] <= index[None, :, None]
1287
+ )
1288
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1289
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1290
+
1291
+ head_p = torch.softmax(head_logits, dim=-1)
1292
+
1293
+ return head_p
1294
+
1295
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1296
+ """Parse input sentence.
1297
+
1298
+ Args:
1299
+ x: input tokens (required).
1300
+ pos: position for each token (optional).
1301
+ Returns:
1302
+ distance: syntactic distance
1303
+ height: syntactic height
1304
+ """
1305
+
1306
+ mask = x != self.pad
1307
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1308
+
1309
+ if embs is None:
1310
+ h = self.roberta.embeddings(x)
1311
+ else:
1312
+ h = embs
1313
+
1314
+ if n_cntxt_layers > 0:
1315
+ if n_cntxt_layers == 1:
1316
+ parser_layers = self.parser_layers_1
1317
+ height_ff = self.height_ff_1
1318
+ distance_ff = self.distance_ff_1
1319
+ elif n_cntxt_layers == 2:
1320
+ parser_layers = self.parser_layers_2
1321
+ height_ff = self.height_ff_2
1322
+ distance_ff = self.distance_ff_2
1323
+ for i in range(int(self.config.n_parser_layers/2)):
1324
+ h = h.masked_fill(~mask[:, :, None], 0)
1325
+ h = parser_layers[i](h)
1326
+
1327
+ height = height_ff(h).squeeze(-1)
1328
+ height.masked_fill_(~mask, -1e9)
1329
+
1330
+ distance = distance_ff(h).squeeze(-1)
1331
+ distance.masked_fill_(~mask_shifted, 1e9)
1332
+
1333
+ # Calbrating the distance and height to the same level
1334
+ length = distance.size(1)
1335
+ height_max = height[:, None, :].expand(-1, length, -1)
1336
+ height_max = torch.cummax(
1337
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1338
+ )[0].triu(0)
1339
+
1340
+ margin_left = torch.relu(
1341
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1342
+ )
1343
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1344
+ margin = torch.where(
1345
+ margin_left > margin_right, margin_right, margin_left
1346
+ ).triu(0)
1347
+
1348
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1349
+ margin.masked_fill_(~margin_mask, 0)
1350
+ margin = margin.max()
1351
+
1352
+ distance = distance - margin
1353
+ else:
1354
+ for i in range(self.config.n_parser_layers):
1355
+ h = h.masked_fill(~mask[:, :, None], 0)
1356
+ h = self.parser_layers[i](h)
1357
+
1358
+ height = self.height_ff(h).squeeze(-1)
1359
+ height.masked_fill_(~mask, -1e9)
1360
+
1361
+ distance = self.distance_ff(h).squeeze(-1)
1362
+ distance.masked_fill_(~mask_shifted, 1e9)
1363
+
1364
+ # Calbrating the distance and height to the same level
1365
+ length = distance.size(1)
1366
+ height_max = height[:, None, :].expand(-1, length, -1)
1367
+ height_max = torch.cummax(
1368
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1369
+ )[0].triu(0)
1370
+
1371
+ margin_left = torch.relu(
1372
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1373
+ )
1374
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1375
+ margin = torch.where(
1376
+ margin_left > margin_right, margin_right, margin_left
1377
+ ).triu(0)
1378
+
1379
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1380
+ margin.masked_fill_(~margin_mask, 0)
1381
+ margin = margin.max()
1382
+
1383
+ distance = distance - margin
1384
+
1385
+ return distance, height
1386
+
1387
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1388
+ """Compute head and cibling distribution for each token."""
1389
+
1390
+ bsz, length = x.size()
1391
+
1392
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1393
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1394
+
1395
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1396
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1397
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1398
+ head = head.masked_fill(eye, 0)
1399
+ child = head.transpose(1, 2)
1400
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1401
+
1402
+ rel_list = []
1403
+ if "head" in self.config.relations:
1404
+ rel_list.append(head)
1405
+ if "child" in self.config.relations:
1406
+ rel_list.append(child)
1407
+ if "cibling" in self.config.relations:
1408
+ rel_list.append(cibling)
1409
+
1410
+ rel = torch.stack(rel_list, dim=1)
1411
+
1412
+ if n_cntxt_layers > 0:
1413
+ if n_cntxt_layers == 1:
1414
+ rel_weight = self.rel_weight_1
1415
+ elif n_cntxt_layers == 2:
1416
+ rel_weight = self.rel_weight_2
1417
+ else:
1418
+ rel_weight = self.rel_weight
1419
+
1420
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1421
+
1422
+ if n_cntxt_layers == 1:
1423
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1424
+ else:
1425
+ num_layers = self.roberta.config.num_hidden_layers
1426
+
1427
+ att_mask = dep.reshape(
1428
+ num_layers,
1429
+ bsz,
1430
+ self.config.num_attention_heads,
1431
+ length,
1432
+ length,
1433
+ )
1434
+
1435
+ return att_mask, cibling, head, block
1436
+
1437
+ def forward(
1438
+ self,
1439
+ input_ids: Optional[torch.LongTensor] = None,
1440
+ attention_mask: Optional[torch.FloatTensor] = None,
1441
+ token_type_ids: Optional[torch.LongTensor] = None,
1442
+ position_ids: Optional[torch.LongTensor] = None,
1443
+ head_mask: Optional[torch.FloatTensor] = None,
1444
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1445
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
1446
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
1447
+ labels: Optional[torch.LongTensor] = None,
1448
+ output_attentions: Optional[bool] = None,
1449
+ output_hidden_states: Optional[bool] = None,
1450
+ return_dict: Optional[bool] = None,
1451
+ ) -> Union[Tuple, MaskedLMOutput]:
1452
+ r"""
1453
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1454
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
1455
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
1456
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
1457
+ kwargs (`Dict[str, any]`, optional, defaults to *{}*):
1458
+ Used to hide legacy arguments that have been deprecated.
1459
+ """
1460
+ return_dict = (
1461
+ return_dict if return_dict is not None else self.config.use_return_dict
1462
+ )
1463
+
1464
+
1465
+ if self.config.n_cntxt_layers > 0:
1466
+ cntxt_outputs = self.cntxt_layers(
1467
+ input_ids,
1468
+ attention_mask=attention_mask,
1469
+ token_type_ids=token_type_ids,
1470
+ position_ids=position_ids,
1471
+ head_mask=head_mask,
1472
+ inputs_embeds=inputs_embeds,
1473
+ encoder_hidden_states=encoder_hidden_states,
1474
+ encoder_attention_mask=encoder_attention_mask,
1475
+ output_attentions=output_attentions,
1476
+ output_hidden_states=output_hidden_states,
1477
+ return_dict=return_dict)
1478
+
1479
+
1480
+ if self.config.n_cntxt_layers_2 > 0:
1481
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1482
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1483
+
1484
+ cntxt_outputs_2 = self.cntxt_layers_2(
1485
+ input_ids,
1486
+ attention_mask=attention_mask,
1487
+ token_type_ids=token_type_ids,
1488
+ position_ids=position_ids,
1489
+ head_mask=head_mask,
1490
+ inputs_embeds=inputs_embeds,
1491
+ encoder_hidden_states=encoder_hidden_states,
1492
+ encoder_attention_mask=encoder_attention_mask,
1493
+ output_attentions=output_attentions,
1494
+ output_hidden_states=output_hidden_states,
1495
+ return_dict=return_dict,
1496
+ parser_att_mask=att_mask_1)
1497
+
1498
+ sequence_output = cntxt_outputs_2[0]
1499
+
1500
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
1501
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
1502
+
1503
+ elif self.config.n_cntxt_layers > 0:
1504
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
1505
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1506
+ else:
1507
+ distance, height = self.parse(input_ids)
1508
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1509
+
1510
+ outputs = self.roberta(
1511
+ input_ids,
1512
+ attention_mask=attention_mask,
1513
+ token_type_ids=token_type_ids,
1514
+ position_ids=position_ids,
1515
+ head_mask=head_mask,
1516
+ inputs_embeds=inputs_embeds,
1517
+ encoder_hidden_states=encoder_hidden_states,
1518
+ encoder_attention_mask=encoder_attention_mask,
1519
+ output_attentions=output_attentions,
1520
+ output_hidden_states=output_hidden_states,
1521
+ return_dict=return_dict,
1522
+ parser_att_mask=att_mask,
1523
+ )
1524
+ sequence_output = outputs[0]
1525
+ prediction_scores = self.lm_head(sequence_output)
1526
+
1527
+ masked_lm_loss = None
1528
+ if labels is not None:
1529
+ loss_fct = CrossEntropyLoss()
1530
+ masked_lm_loss = loss_fct(
1531
+ prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)
1532
+ )
1533
+
1534
+ if not return_dict:
1535
+ output = (prediction_scores,) + outputs[2:]
1536
+ return (
1537
+ ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
1538
+ )
1539
+
1540
+ return MaskedLMOutput(
1541
+ loss=masked_lm_loss,
1542
+ logits=prediction_scores,
1543
+ hidden_states=outputs.hidden_states,
1544
+ attentions=outputs.attentions,
1545
+ )
1546
+
1547
+
1548
+ class RobertaLMHead(nn.Module):
1549
+ """Roberta Head for masked language modeling."""
1550
+
1551
+ def __init__(self, config):
1552
+ super().__init__()
1553
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
1554
+ self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
1555
+
1556
+ self.decoder = nn.Linear(config.hidden_size, config.vocab_size)
1557
+ self.bias = nn.Parameter(torch.zeros(config.vocab_size))
1558
+ self.decoder.bias = self.bias
1559
+
1560
+ def forward(self, features, **kwargs):
1561
+ x = self.dense(features)
1562
+ x = gelu(x)
1563
+ x = self.layer_norm(x)
1564
+
1565
+ # project back to size of vocabulary with bias
1566
+ x = self.decoder(x)
1567
+
1568
+ return x
1569
+
1570
+ def _tie_weights(self):
1571
+ # To tie those two weights if they get disconnected (on TPU or when the bias is resized)
1572
+ self.bias = self.decoder.bias
1573
+
1574
+
1575
+ class StructRobertaForSequenceClassification(RobertaPreTrainedModel):
1576
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
1577
+
1578
+ def __init__(self, config):
1579
+ super().__init__(config)
1580
+ self.num_labels = config.num_labels
1581
+ self.config = config
1582
+
1583
+ if config.n_cntxt_layers > 0:
1584
+ config_cntxt = copy.deepcopy(config)
1585
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1586
+
1587
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1588
+
1589
+ if config.n_cntxt_layers_2 > 0:
1590
+ self.parser_layers_1 = nn.ModuleList(
1591
+ [
1592
+ nn.Sequential(
1593
+ Conv1d(config.hidden_size, config.conv_size),
1594
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1595
+ nn.Tanh(),
1596
+ )
1597
+ for i in range(int(config.n_parser_layers/2))
1598
+ ]
1599
+ )
1600
+
1601
+ self.distance_ff_1 = nn.Sequential(
1602
+ Conv1d(config.hidden_size, 2),
1603
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1604
+ nn.Tanh(),
1605
+ nn.Linear(config.hidden_size, 1),
1606
+ )
1607
+
1608
+ self.height_ff_1 = nn.Sequential(
1609
+ nn.Linear(config.hidden_size, config.hidden_size),
1610
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1611
+ nn.Tanh(),
1612
+ nn.Linear(config.hidden_size, 1),
1613
+ )
1614
+
1615
+ n_rel = len(config.relations)
1616
+ self._rel_weight_1 = nn.Parameter(
1617
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1618
+ )
1619
+ self._rel_weight_1.data.normal_(0, 0.1)
1620
+
1621
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1622
+
1623
+ config_cntxt_2 = copy.deepcopy(config)
1624
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1625
+
1626
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1627
+
1628
+
1629
+ self.parser_layers_2 = nn.ModuleList(
1630
+ [
1631
+ nn.Sequential(
1632
+ Conv1d(config.hidden_size, config.conv_size),
1633
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1634
+ nn.Tanh(),
1635
+ )
1636
+ for i in range(int(config.n_parser_layers/2))
1637
+ ]
1638
+ )
1639
+
1640
+ self.distance_ff_2 = nn.Sequential(
1641
+ Conv1d(config.hidden_size, 2),
1642
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1643
+ nn.Tanh(),
1644
+ nn.Linear(config.hidden_size, 1),
1645
+ )
1646
+
1647
+ self.height_ff_2 = nn.Sequential(
1648
+ nn.Linear(config.hidden_size, config.hidden_size),
1649
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1650
+ nn.Tanh(),
1651
+ nn.Linear(config.hidden_size, 1),
1652
+ )
1653
+
1654
+ n_rel = len(config.relations)
1655
+ self._rel_weight_2 = nn.Parameter(
1656
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1657
+ )
1658
+ self._rel_weight_2.data.normal_(0, 0.1)
1659
+
1660
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1661
+
1662
+ else:
1663
+ self.parser_layers = nn.ModuleList(
1664
+ [
1665
+ nn.Sequential(
1666
+ Conv1d(config.hidden_size, config.conv_size),
1667
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1668
+ nn.Tanh(),
1669
+ )
1670
+ for i in range(config.n_parser_layers)
1671
+ ]
1672
+ )
1673
+
1674
+ self.distance_ff = nn.Sequential(
1675
+ Conv1d(config.hidden_size, 2),
1676
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1677
+ nn.Tanh(),
1678
+ nn.Linear(config.hidden_size, 1),
1679
+ )
1680
+
1681
+ self.height_ff = nn.Sequential(
1682
+ nn.Linear(config.hidden_size, config.hidden_size),
1683
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1684
+ nn.Tanh(),
1685
+ nn.Linear(config.hidden_size, 1),
1686
+ )
1687
+
1688
+ n_rel = len(config.relations)
1689
+ self._rel_weight = nn.Parameter(
1690
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1691
+ )
1692
+ self._rel_weight.data.normal_(0, 0.1)
1693
+
1694
+ self._scaler = nn.Parameter(torch.zeros(2))
1695
+
1696
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1697
+
1698
+ if config.n_cntxt_layers > 0:
1699
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1700
+ if config.n_cntxt_layers_2 > 0:
1701
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1702
+
1703
+
1704
+ self.pad = config.pad_token_id
1705
+ self.classifier = RobertaClassificationHead(config)
1706
+
1707
+ # Initialize weights and apply final processing
1708
+ self.post_init()
1709
+
1710
+
1711
+ @property
1712
+ def scaler(self):
1713
+ return self._scaler.exp()
1714
+
1715
+ @property
1716
+ def scaler_1(self):
1717
+ return self._scaler_1.exp()
1718
+
1719
+ @property
1720
+ def scaler_2(self):
1721
+ return self._scaler_2.exp()
1722
+
1723
+ @property
1724
+ def rel_weight(self):
1725
+ if self.config.weight_act == "sigmoid":
1726
+ return torch.sigmoid(self._rel_weight)
1727
+ elif self.config.weight_act == "softmax":
1728
+ return torch.softmax(self._rel_weight, dim=-1)
1729
+
1730
+ @property
1731
+ def rel_weight_1(self):
1732
+ if self.config.weight_act == "sigmoid":
1733
+ return torch.sigmoid(self._rel_weight_1)
1734
+ elif self.config.weight_act == "softmax":
1735
+ return torch.softmax(self._rel_weight_1, dim=-1)
1736
+
1737
+
1738
+ @property
1739
+ def rel_weight_2(self):
1740
+ if self.config.weight_act == "sigmoid":
1741
+ return torch.sigmoid(self._rel_weight_2)
1742
+ elif self.config.weight_act == "softmax":
1743
+ return torch.softmax(self._rel_weight_2, dim=-1)
1744
+
1745
+
1746
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1747
+ """Compute constituents from distance and height."""
1748
+
1749
+ if n_cntxt_layers>0:
1750
+ if n_cntxt_layers == 1:
1751
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1752
+ elif n_cntxt_layers == 2:
1753
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1754
+ else:
1755
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1756
+
1757
+ gamma = torch.sigmoid(-beta_logits)
1758
+ ones = torch.ones_like(gamma)
1759
+
1760
+ block_mask_left = cummin(
1761
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1762
+ )
1763
+ block_mask_left = block_mask_left - F.pad(
1764
+ block_mask_left[:, :, :-1], (1, 0), value=0
1765
+ )
1766
+ block_mask_left.tril_(0)
1767
+
1768
+ block_mask_right = cummin(
1769
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1770
+ )
1771
+ block_mask_right = block_mask_right - F.pad(
1772
+ block_mask_right[:, :, 1:], (0, 1), value=0
1773
+ )
1774
+ block_mask_right.triu_(0)
1775
+
1776
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1777
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1778
+ block_mask_right, reverse=True
1779
+ ).triu(1)
1780
+
1781
+ return block_p, block
1782
+
1783
+ def compute_head(self, height, n_cntxt_layers=0):
1784
+ """Estimate head for each constituent."""
1785
+
1786
+ _, length = height.size()
1787
+ if n_cntxt_layers>0:
1788
+ if n_cntxt_layers == 1:
1789
+ head_logits = height * self.scaler_1[1]
1790
+ elif n_cntxt_layers == 2:
1791
+ head_logits = height * self.scaler_2[1]
1792
+ else:
1793
+ head_logits = height * self.scaler[1]
1794
+ index = torch.arange(length, device=height.device)
1795
+
1796
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1797
+ index[None, None, :] <= index[None, :, None]
1798
+ )
1799
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1800
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1801
+
1802
+ head_p = torch.softmax(head_logits, dim=-1)
1803
+
1804
+ return head_p
1805
+
1806
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1807
+ """Parse input sentence.
1808
+
1809
+ Args:
1810
+ x: input tokens (required).
1811
+ pos: position for each token (optional).
1812
+ Returns:
1813
+ distance: syntactic distance
1814
+ height: syntactic height
1815
+ """
1816
+
1817
+ mask = x != self.pad
1818
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1819
+
1820
+ if embs is None:
1821
+ h = self.roberta.embeddings(x)
1822
+ else:
1823
+ h = embs
1824
+
1825
+ if n_cntxt_layers > 0:
1826
+ if n_cntxt_layers == 1:
1827
+ parser_layers = self.parser_layers_1
1828
+ height_ff = self.height_ff_1
1829
+ distance_ff = self.distance_ff_1
1830
+ elif n_cntxt_layers == 2:
1831
+ parser_layers = self.parser_layers_2
1832
+ height_ff = self.height_ff_2
1833
+ distance_ff = self.distance_ff_2
1834
+ for i in range(int(self.config.n_parser_layers/2)):
1835
+ h = h.masked_fill(~mask[:, :, None], 0)
1836
+ h = parser_layers[i](h)
1837
+
1838
+ height = height_ff(h).squeeze(-1)
1839
+ height.masked_fill_(~mask, -1e9)
1840
+
1841
+ distance = distance_ff(h).squeeze(-1)
1842
+ distance.masked_fill_(~mask_shifted, 1e9)
1843
+
1844
+ # Calbrating the distance and height to the same level
1845
+ length = distance.size(1)
1846
+ height_max = height[:, None, :].expand(-1, length, -1)
1847
+ height_max = torch.cummax(
1848
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1849
+ )[0].triu(0)
1850
+
1851
+ margin_left = torch.relu(
1852
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1853
+ )
1854
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1855
+ margin = torch.where(
1856
+ margin_left > margin_right, margin_right, margin_left
1857
+ ).triu(0)
1858
+
1859
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1860
+ margin.masked_fill_(~margin_mask, 0)
1861
+ margin = margin.max()
1862
+
1863
+ distance = distance - margin
1864
+ else:
1865
+ for i in range(self.config.n_parser_layers):
1866
+ h = h.masked_fill(~mask[:, :, None], 0)
1867
+ h = self.parser_layers[i](h)
1868
+
1869
+ height = self.height_ff(h).squeeze(-1)
1870
+ height.masked_fill_(~mask, -1e9)
1871
+
1872
+ distance = self.distance_ff(h).squeeze(-1)
1873
+ distance.masked_fill_(~mask_shifted, 1e9)
1874
+
1875
+ # Calbrating the distance and height to the same level
1876
+ length = distance.size(1)
1877
+ height_max = height[:, None, :].expand(-1, length, -1)
1878
+ height_max = torch.cummax(
1879
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1880
+ )[0].triu(0)
1881
+
1882
+ margin_left = torch.relu(
1883
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1884
+ )
1885
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1886
+ margin = torch.where(
1887
+ margin_left > margin_right, margin_right, margin_left
1888
+ ).triu(0)
1889
+
1890
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1891
+ margin.masked_fill_(~margin_mask, 0)
1892
+ margin = margin.max()
1893
+
1894
+ distance = distance - margin
1895
+
1896
+ return distance, height
1897
+
1898
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1899
+ """Compute head and cibling distribution for each token."""
1900
+
1901
+ bsz, length = x.size()
1902
+
1903
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1904
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1905
+
1906
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1907
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1908
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1909
+ head = head.masked_fill(eye, 0)
1910
+ child = head.transpose(1, 2)
1911
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1912
+
1913
+ rel_list = []
1914
+ if "head" in self.config.relations:
1915
+ rel_list.append(head)
1916
+ if "child" in self.config.relations:
1917
+ rel_list.append(child)
1918
+ if "cibling" in self.config.relations:
1919
+ rel_list.append(cibling)
1920
+
1921
+ rel = torch.stack(rel_list, dim=1)
1922
+
1923
+ if n_cntxt_layers > 0:
1924
+ if n_cntxt_layers == 1:
1925
+ rel_weight = self.rel_weight_1
1926
+ elif n_cntxt_layers == 2:
1927
+ rel_weight = self.rel_weight_2
1928
+ else:
1929
+ rel_weight = self.rel_weight
1930
+
1931
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1932
+
1933
+ if n_cntxt_layers == 1:
1934
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1935
+ else:
1936
+ num_layers = self.roberta.config.num_hidden_layers
1937
+
1938
+ att_mask = dep.reshape(
1939
+ num_layers,
1940
+ bsz,
1941
+ self.config.num_attention_heads,
1942
+ length,
1943
+ length,
1944
+ )
1945
+
1946
+ return att_mask, cibling, head, block
1947
+
1948
+ def forward(
1949
+ self,
1950
+ input_ids: Optional[torch.LongTensor] = None,
1951
+ attention_mask: Optional[torch.FloatTensor] = None,
1952
+ token_type_ids: Optional[torch.LongTensor] = None,
1953
+ position_ids: Optional[torch.LongTensor] = None,
1954
+ head_mask: Optional[torch.FloatTensor] = None,
1955
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1956
+ labels: Optional[torch.LongTensor] = None,
1957
+ output_attentions: Optional[bool] = None,
1958
+ output_hidden_states: Optional[bool] = None,
1959
+ return_dict: Optional[bool] = None,
1960
+ ) -> Union[Tuple, SequenceClassifierOutput]:
1961
+ r"""
1962
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1963
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1964
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1965
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1966
+ """
1967
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1968
+
1969
+ if self.config.n_cntxt_layers > 0:
1970
+ cntxt_outputs = self.cntxt_layers(
1971
+ input_ids,
1972
+ attention_mask=attention_mask,
1973
+ token_type_ids=token_type_ids,
1974
+ position_ids=position_ids,
1975
+ head_mask=head_mask,
1976
+ inputs_embeds=inputs_embeds,
1977
+ output_attentions=output_attentions,
1978
+ output_hidden_states=output_hidden_states,
1979
+ return_dict=return_dict)
1980
+
1981
+
1982
+ if self.config.n_cntxt_layers_2 > 0:
1983
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1984
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1985
+
1986
+ cntxt_outputs_2 = self.cntxt_layers_2(
1987
+ input_ids,
1988
+ attention_mask=attention_mask,
1989
+ token_type_ids=token_type_ids,
1990
+ position_ids=position_ids,
1991
+ head_mask=head_mask,
1992
+ inputs_embeds=inputs_embeds,
1993
+ output_attentions=output_attentions,
1994
+ output_hidden_states=output_hidden_states,
1995
+ return_dict=return_dict,
1996
+ parser_att_mask=att_mask_1)
1997
+
1998
+ sequence_output = cntxt_outputs_2[0]
1999
+
2000
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
2001
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
2002
+
2003
+ elif self.config.n_cntxt_layers > 0:
2004
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
2005
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2006
+ else:
2007
+ distance, height = self.parse(input_ids)
2008
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2009
+
2010
+ outputs = self.roberta(
2011
+ input_ids,
2012
+ attention_mask=attention_mask,
2013
+ token_type_ids=token_type_ids,
2014
+ position_ids=position_ids,
2015
+ head_mask=head_mask,
2016
+ inputs_embeds=inputs_embeds,
2017
+ output_attentions=output_attentions,
2018
+ output_hidden_states=output_hidden_states,
2019
+ return_dict=return_dict,
2020
+ parser_att_mask=att_mask,
2021
+ )
2022
+ sequence_output = outputs[0]
2023
+ logits = self.classifier(sequence_output)
2024
+
2025
+ loss = None
2026
+ if labels is not None:
2027
+ if self.config.problem_type is None:
2028
+ if self.num_labels == 1:
2029
+ self.config.problem_type = "regression"
2030
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
2031
+ self.config.problem_type = "single_label_classification"
2032
+ else:
2033
+ self.config.problem_type = "multi_label_classification"
2034
+
2035
+ if self.config.problem_type == "regression":
2036
+ loss_fct = MSELoss()
2037
+ if self.num_labels == 1:
2038
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
2039
+ else:
2040
+ loss = loss_fct(logits, labels)
2041
+ elif self.config.problem_type == "single_label_classification":
2042
+ loss_fct = CrossEntropyLoss()
2043
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
2044
+ elif self.config.problem_type == "multi_label_classification":
2045
+ loss_fct = BCEWithLogitsLoss()
2046
+ loss = loss_fct(logits, labels)
2047
+
2048
+ if not return_dict:
2049
+ output = (logits,) + outputs[2:]
2050
+ return ((loss,) + output) if loss is not None else output
2051
+
2052
+ return SequenceClassifierOutput(
2053
+ loss=loss,
2054
+ logits=logits,
2055
+ hidden_states=outputs.hidden_states,
2056
+ attentions=outputs.attentions,
2057
+ )
2058
+
2059
+
2060
+ class RobertaClassificationHead(nn.Module):
2061
+ """Head for sentence-level classification tasks."""
2062
+
2063
+ def __init__(self, config):
2064
+ super().__init__()
2065
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
2066
+ classifier_dropout = (
2067
+ config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
2068
+ )
2069
+ self.dropout = nn.Dropout(classifier_dropout)
2070
+ self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
2071
+
2072
+ def forward(self, features, **kwargs):
2073
+ x = features[:, 0, :] # take <s> token (equiv. to [CLS])
2074
+ x = self.dropout(x)
2075
+ x = self.dense(x)
2076
+ x = torch.tanh(x)
2077
+ x = self.dropout(x)
2078
+ x = self.out_proj(x)
2079
+ return x
2080
+
2081
+
2082
+ def create_position_ids_from_input_ids(
2083
+ input_ids, padding_idx, past_key_values_length=0
2084
+ ):
2085
+ """
2086
+ Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
2087
+ are ignored. This is modified from fairseq's `utils.make_positions`.
2088
+
2089
+ Args:
2090
+ x: torch.Tensor x:
2091
+
2092
+ Returns: torch.Tensor
2093
+ """
2094
+ # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
2095
+ mask = input_ids.ne(padding_idx).int()
2096
+ incremental_indices = (
2097
+ torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length
2098
+ ) * mask
2099
+ return incremental_indices.long() + padding_idx
2100
+
2101
+
2102
+ def cumprod(x, reverse=False, exclusive=False):
2103
+ """cumulative product."""
2104
+ if reverse:
2105
+ x = x.flip([-1])
2106
+
2107
+ if exclusive:
2108
+ x = F.pad(x[:, :, :-1], (1, 0), value=1)
2109
+
2110
+ cx = x.cumprod(-1)
2111
+
2112
+ if reverse:
2113
+ cx = cx.flip([-1])
2114
+ return cx
2115
+
2116
+
2117
+ def cumsum(x, reverse=False, exclusive=False):
2118
+ """cumulative sum."""
2119
+ bsz, _, length = x.size()
2120
+ device = x.device
2121
+ if reverse:
2122
+ if exclusive:
2123
+ w = torch.ones([bsz, length, length], device=device).tril(-1)
2124
+ else:
2125
+ w = torch.ones([bsz, length, length], device=device).tril(0)
2126
+ cx = torch.bmm(x, w)
2127
+ else:
2128
+ if exclusive:
2129
+ w = torch.ones([bsz, length, length], device=device).triu(1)
2130
+ else:
2131
+ w = torch.ones([bsz, length, length], device=device).triu(0)
2132
+ cx = torch.bmm(x, w)
2133
+ return cx
2134
+
2135
+
2136
+ def cummin(x, reverse=False, exclusive=False, max_value=1e9):
2137
+ """cumulative min."""
2138
+ if reverse:
2139
+ if exclusive:
2140
+ x = F.pad(x[:, :, 1:], (0, 1), value=max_value)
2141
+ x = x.flip([-1]).cummin(-1)[0].flip([-1])
2142
+ else:
2143
+ if exclusive:
2144
+ x = F.pad(x[:, :, :-1], (1, 0), value=max_value)
2145
+ x = x.cummin(-1)[0]
2146
+ return x
finetune/cola/predict_results.txt ADDED
@@ -0,0 +1,1020 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ index prediction
2
+ 0 1
3
+ 1 1
4
+ 2 0
5
+ 3 1
6
+ 4 0
7
+ 5 1
8
+ 6 1
9
+ 7 1
10
+ 8 1
11
+ 9 1
12
+ 10 1
13
+ 11 1
14
+ 12 1
15
+ 13 0
16
+ 14 1
17
+ 15 1
18
+ 16 1
19
+ 17 1
20
+ 18 0
21
+ 19 1
22
+ 20 1
23
+ 21 1
24
+ 22 1
25
+ 23 1
26
+ 24 1
27
+ 25 1
28
+ 26 1
29
+ 27 1
30
+ 28 0
31
+ 29 1
32
+ 30 1
33
+ 31 0
34
+ 32 1
35
+ 33 1
36
+ 34 0
37
+ 35 0
38
+ 36 1
39
+ 37 1
40
+ 38 1
41
+ 39 1
42
+ 40 0
43
+ 41 0
44
+ 42 1
45
+ 43 1
46
+ 44 0
47
+ 45 0
48
+ 46 1
49
+ 47 0
50
+ 48 1
51
+ 49 1
52
+ 50 1
53
+ 51 1
54
+ 52 0
55
+ 53 0
56
+ 54 0
57
+ 55 1
58
+ 56 1
59
+ 57 1
60
+ 58 1
61
+ 59 0
62
+ 60 0
63
+ 61 1
64
+ 62 0
65
+ 63 1
66
+ 64 1
67
+ 65 0
68
+ 66 1
69
+ 67 0
70
+ 68 1
71
+ 69 1
72
+ 70 1
73
+ 71 1
74
+ 72 1
75
+ 73 0
76
+ 74 1
77
+ 75 0
78
+ 76 1
79
+ 77 1
80
+ 78 1
81
+ 79 1
82
+ 80 1
83
+ 81 1
84
+ 82 1
85
+ 83 0
86
+ 84 1
87
+ 85 0
88
+ 86 0
89
+ 87 1
90
+ 88 0
91
+ 89 0
92
+ 90 0
93
+ 91 0
94
+ 92 0
95
+ 93 1
96
+ 94 1
97
+ 95 1
98
+ 96 0
99
+ 97 1
100
+ 98 1
101
+ 99 0
102
+ 100 1
103
+ 101 0
104
+ 102 1
105
+ 103 1
106
+ 104 1
107
+ 105 1
108
+ 106 1
109
+ 107 1
110
+ 108 1
111
+ 109 0
112
+ 110 1
113
+ 111 0
114
+ 112 1
115
+ 113 1
116
+ 114 1
117
+ 115 1
118
+ 116 1
119
+ 117 1
120
+ 118 0
121
+ 119 1
122
+ 120 1
123
+ 121 1
124
+ 122 1
125
+ 123 1
126
+ 124 1
127
+ 125 1
128
+ 126 1
129
+ 127 0
130
+ 128 0
131
+ 129 1
132
+ 130 1
133
+ 131 1
134
+ 132 1
135
+ 133 1
136
+ 134 1
137
+ 135 1
138
+ 136 1
139
+ 137 1
140
+ 138 1
141
+ 139 1
142
+ 140 1
143
+ 141 1
144
+ 142 1
145
+ 143 1
146
+ 144 0
147
+ 145 1
148
+ 146 0
149
+ 147 1
150
+ 148 1
151
+ 149 1
152
+ 150 1
153
+ 151 0
154
+ 152 1
155
+ 153 1
156
+ 154 1
157
+ 155 1
158
+ 156 1
159
+ 157 1
160
+ 158 1
161
+ 159 0
162
+ 160 1
163
+ 161 1
164
+ 162 0
165
+ 163 1
166
+ 164 1
167
+ 165 1
168
+ 166 0
169
+ 167 1
170
+ 168 0
171
+ 169 1
172
+ 170 1
173
+ 171 0
174
+ 172 1
175
+ 173 0
176
+ 174 1
177
+ 175 1
178
+ 176 1
179
+ 177 1
180
+ 178 1
181
+ 179 1
182
+ 180 0
183
+ 181 1
184
+ 182 1
185
+ 183 1
186
+ 184 1
187
+ 185 1
188
+ 186 1
189
+ 187 0
190
+ 188 0
191
+ 189 1
192
+ 190 0
193
+ 191 1
194
+ 192 1
195
+ 193 1
196
+ 194 1
197
+ 195 0
198
+ 196 1
199
+ 197 0
200
+ 198 1
201
+ 199 1
202
+ 200 1
203
+ 201 1
204
+ 202 1
205
+ 203 1
206
+ 204 1
207
+ 205 1
208
+ 206 1
209
+ 207 1
210
+ 208 1
211
+ 209 1
212
+ 210 1
213
+ 211 1
214
+ 212 1
215
+ 213 0
216
+ 214 1
217
+ 215 1
218
+ 216 0
219
+ 217 1
220
+ 218 1
221
+ 219 1
222
+ 220 0
223
+ 221 1
224
+ 222 1
225
+ 223 1
226
+ 224 1
227
+ 225 1
228
+ 226 1
229
+ 227 1
230
+ 228 1
231
+ 229 1
232
+ 230 1
233
+ 231 1
234
+ 232 0
235
+ 233 1
236
+ 234 1
237
+ 235 1
238
+ 236 1
239
+ 237 1
240
+ 238 1
241
+ 239 1
242
+ 240 1
243
+ 241 1
244
+ 242 0
245
+ 243 0
246
+ 244 0
247
+ 245 1
248
+ 246 1
249
+ 247 1
250
+ 248 0
251
+ 249 0
252
+ 250 1
253
+ 251 1
254
+ 252 1
255
+ 253 1
256
+ 254 1
257
+ 255 0
258
+ 256 1
259
+ 257 0
260
+ 258 1
261
+ 259 1
262
+ 260 1
263
+ 261 1
264
+ 262 0
265
+ 263 0
266
+ 264 0
267
+ 265 1
268
+ 266 0
269
+ 267 1
270
+ 268 1
271
+ 269 0
272
+ 270 1
273
+ 271 0
274
+ 272 1
275
+ 273 0
276
+ 274 1
277
+ 275 1
278
+ 276 0
279
+ 277 1
280
+ 278 1
281
+ 279 0
282
+ 280 1
283
+ 281 1
284
+ 282 1
285
+ 283 0
286
+ 284 1
287
+ 285 1
288
+ 286 0
289
+ 287 0
290
+ 288 1
291
+ 289 1
292
+ 290 1
293
+ 291 1
294
+ 292 1
295
+ 293 1
296
+ 294 1
297
+ 295 1
298
+ 296 0
299
+ 297 1
300
+ 298 1
301
+ 299 1
302
+ 300 1
303
+ 301 1
304
+ 302 1
305
+ 303 1
306
+ 304 0
307
+ 305 1
308
+ 306 0
309
+ 307 1
310
+ 308 1
311
+ 309 1
312
+ 310 1
313
+ 311 1
314
+ 312 1
315
+ 313 1
316
+ 314 1
317
+ 315 1
318
+ 316 1
319
+ 317 1
320
+ 318 1
321
+ 319 1
322
+ 320 1
323
+ 321 1
324
+ 322 0
325
+ 323 0
326
+ 324 0
327
+ 325 1
328
+ 326 1
329
+ 327 1
330
+ 328 0
331
+ 329 1
332
+ 330 1
333
+ 331 1
334
+ 332 0
335
+ 333 1
336
+ 334 1
337
+ 335 1
338
+ 336 1
339
+ 337 1
340
+ 338 0
341
+ 339 1
342
+ 340 1
343
+ 341 1
344
+ 342 1
345
+ 343 1
346
+ 344 1
347
+ 345 0
348
+ 346 1
349
+ 347 1
350
+ 348 1
351
+ 349 1
352
+ 350 1
353
+ 351 1
354
+ 352 1
355
+ 353 1
356
+ 354 1
357
+ 355 1
358
+ 356 1
359
+ 357 1
360
+ 358 1
361
+ 359 1
362
+ 360 1
363
+ 361 1
364
+ 362 0
365
+ 363 1
366
+ 364 1
367
+ 365 1
368
+ 366 1
369
+ 367 1
370
+ 368 1
371
+ 369 1
372
+ 370 1
373
+ 371 1
374
+ 372 1
375
+ 373 1
376
+ 374 1
377
+ 375 1
378
+ 376 1
379
+ 377 1
380
+ 378 0
381
+ 379 1
382
+ 380 1
383
+ 381 1
384
+ 382 1
385
+ 383 1
386
+ 384 1
387
+ 385 1
388
+ 386 1
389
+ 387 1
390
+ 388 0
391
+ 389 1
392
+ 390 1
393
+ 391 1
394
+ 392 0
395
+ 393 0
396
+ 394 1
397
+ 395 1
398
+ 396 1
399
+ 397 1
400
+ 398 1
401
+ 399 1
402
+ 400 1
403
+ 401 0
404
+ 402 1
405
+ 403 1
406
+ 404 0
407
+ 405 1
408
+ 406 1
409
+ 407 1
410
+ 408 1
411
+ 409 0
412
+ 410 1
413
+ 411 1
414
+ 412 1
415
+ 413 1
416
+ 414 1
417
+ 415 1
418
+ 416 1
419
+ 417 1
420
+ 418 1
421
+ 419 1
422
+ 420 1
423
+ 421 1
424
+ 422 0
425
+ 423 1
426
+ 424 1
427
+ 425 1
428
+ 426 1
429
+ 427 1
430
+ 428 0
431
+ 429 1
432
+ 430 0
433
+ 431 0
434
+ 432 1
435
+ 433 1
436
+ 434 1
437
+ 435 1
438
+ 436 1
439
+ 437 1
440
+ 438 1
441
+ 439 1
442
+ 440 1
443
+ 441 1
444
+ 442 1
445
+ 443 1
446
+ 444 0
447
+ 445 0
448
+ 446 1
449
+ 447 1
450
+ 448 0
451
+ 449 1
452
+ 450 0
453
+ 451 0
454
+ 452 0
455
+ 453 0
456
+ 454 1
457
+ 455 1
458
+ 456 1
459
+ 457 1
460
+ 458 1
461
+ 459 0
462
+ 460 1
463
+ 461 1
464
+ 462 1
465
+ 463 0
466
+ 464 1
467
+ 465 0
468
+ 466 1
469
+ 467 0
470
+ 468 1
471
+ 469 1
472
+ 470 1
473
+ 471 1
474
+ 472 1
475
+ 473 1
476
+ 474 0
477
+ 475 1
478
+ 476 1
479
+ 477 1
480
+ 478 1
481
+ 479 1
482
+ 480 1
483
+ 481 0
484
+ 482 1
485
+ 483 0
486
+ 484 0
487
+ 485 1
488
+ 486 1
489
+ 487 1
490
+ 488 0
491
+ 489 1
492
+ 490 1
493
+ 491 1
494
+ 492 1
495
+ 493 0
496
+ 494 1
497
+ 495 1
498
+ 496 1
499
+ 497 1
500
+ 498 1
501
+ 499 1
502
+ 500 1
503
+ 501 1
504
+ 502 1
505
+ 503 1
506
+ 504 0
507
+ 505 0
508
+ 506 0
509
+ 507 1
510
+ 508 0
511
+ 509 0
512
+ 510 1
513
+ 511 1
514
+ 512 1
515
+ 513 1
516
+ 514 1
517
+ 515 1
518
+ 516 1
519
+ 517 1
520
+ 518 1
521
+ 519 1
522
+ 520 1
523
+ 521 1
524
+ 522 1
525
+ 523 1
526
+ 524 1
527
+ 525 1
528
+ 526 1
529
+ 527 1
530
+ 528 1
531
+ 529 1
532
+ 530 1
533
+ 531 1
534
+ 532 1
535
+ 533 1
536
+ 534 1
537
+ 535 1
538
+ 536 1
539
+ 537 1
540
+ 538 1
541
+ 539 1
542
+ 540 1
543
+ 541 1
544
+ 542 1
545
+ 543 1
546
+ 544 1
547
+ 545 0
548
+ 546 1
549
+ 547 1
550
+ 548 1
551
+ 549 1
552
+ 550 1
553
+ 551 1
554
+ 552 1
555
+ 553 0
556
+ 554 1
557
+ 555 1
558
+ 556 1
559
+ 557 1
560
+ 558 1
561
+ 559 1
562
+ 560 1
563
+ 561 1
564
+ 562 1
565
+ 563 1
566
+ 564 1
567
+ 565 1
568
+ 566 1
569
+ 567 1
570
+ 568 1
571
+ 569 1
572
+ 570 0
573
+ 571 1
574
+ 572 1
575
+ 573 1
576
+ 574 1
577
+ 575 1
578
+ 576 1
579
+ 577 1
580
+ 578 1
581
+ 579 1
582
+ 580 1
583
+ 581 1
584
+ 582 1
585
+ 583 1
586
+ 584 1
587
+ 585 1
588
+ 586 1
589
+ 587 1
590
+ 588 1
591
+ 589 1
592
+ 590 1
593
+ 591 1
594
+ 592 1
595
+ 593 0
596
+ 594 1
597
+ 595 1
598
+ 596 1
599
+ 597 1
600
+ 598 1
601
+ 599 0
602
+ 600 0
603
+ 601 1
604
+ 602 0
605
+ 603 0
606
+ 604 0
607
+ 605 0
608
+ 606 0
609
+ 607 0
610
+ 608 1
611
+ 609 1
612
+ 610 1
613
+ 611 1
614
+ 612 1
615
+ 613 1
616
+ 614 1
617
+ 615 1
618
+ 616 0
619
+ 617 0
620
+ 618 1
621
+ 619 1
622
+ 620 1
623
+ 621 0
624
+ 622 1
625
+ 623 1
626
+ 624 0
627
+ 625 1
628
+ 626 1
629
+ 627 1
630
+ 628 1
631
+ 629 1
632
+ 630 1
633
+ 631 1
634
+ 632 1
635
+ 633 1
636
+ 634 1
637
+ 635 1
638
+ 636 1
639
+ 637 1
640
+ 638 1
641
+ 639 1
642
+ 640 0
643
+ 641 1
644
+ 642 1
645
+ 643 1
646
+ 644 0
647
+ 645 0
648
+ 646 1
649
+ 647 0
650
+ 648 0
651
+ 649 1
652
+ 650 1
653
+ 651 0
654
+ 652 1
655
+ 653 1
656
+ 654 0
657
+ 655 0
658
+ 656 1
659
+ 657 1
660
+ 658 1
661
+ 659 1
662
+ 660 1
663
+ 661 1
664
+ 662 1
665
+ 663 1
666
+ 664 1
667
+ 665 0
668
+ 666 1
669
+ 667 0
670
+ 668 1
671
+ 669 1
672
+ 670 1
673
+ 671 0
674
+ 672 1
675
+ 673 0
676
+ 674 1
677
+ 675 1
678
+ 676 1
679
+ 677 1
680
+ 678 0
681
+ 679 1
682
+ 680 1
683
+ 681 0
684
+ 682 0
685
+ 683 1
686
+ 684 1
687
+ 685 1
688
+ 686 1
689
+ 687 1
690
+ 688 1
691
+ 689 1
692
+ 690 1
693
+ 691 1
694
+ 692 1
695
+ 693 1
696
+ 694 1
697
+ 695 1
698
+ 696 1
699
+ 697 1
700
+ 698 1
701
+ 699 1
702
+ 700 1
703
+ 701 0
704
+ 702 1
705
+ 703 0
706
+ 704 0
707
+ 705 0
708
+ 706 1
709
+ 707 0
710
+ 708 0
711
+ 709 1
712
+ 710 1
713
+ 711 1
714
+ 712 1
715
+ 713 1
716
+ 714 1
717
+ 715 1
718
+ 716 1
719
+ 717 1
720
+ 718 0
721
+ 719 1
722
+ 720 0
723
+ 721 1
724
+ 722 1
725
+ 723 1
726
+ 724 1
727
+ 725 0
728
+ 726 1
729
+ 727 1
730
+ 728 1
731
+ 729 1
732
+ 730 1
733
+ 731 1
734
+ 732 1
735
+ 733 0
736
+ 734 1
737
+ 735 0
738
+ 736 1
739
+ 737 1
740
+ 738 1
741
+ 739 1
742
+ 740 1
743
+ 741 0
744
+ 742 1
745
+ 743 1
746
+ 744 1
747
+ 745 1
748
+ 746 1
749
+ 747 1
750
+ 748 1
751
+ 749 1
752
+ 750 1
753
+ 751 1
754
+ 752 1
755
+ 753 1
756
+ 754 1
757
+ 755 1
758
+ 756 1
759
+ 757 1
760
+ 758 1
761
+ 759 0
762
+ 760 1
763
+ 761 1
764
+ 762 1
765
+ 763 1
766
+ 764 1
767
+ 765 0
768
+ 766 0
769
+ 767 1
770
+ 768 1
771
+ 769 1
772
+ 770 1
773
+ 771 1
774
+ 772 1
775
+ 773 1
776
+ 774 1
777
+ 775 1
778
+ 776 1
779
+ 777 1
780
+ 778 1
781
+ 779 1
782
+ 780 1
783
+ 781 1
784
+ 782 1
785
+ 783 1
786
+ 784 1
787
+ 785 1
788
+ 786 0
789
+ 787 0
790
+ 788 1
791
+ 789 1
792
+ 790 1
793
+ 791 1
794
+ 792 1
795
+ 793 1
796
+ 794 0
797
+ 795 1
798
+ 796 1
799
+ 797 1
800
+ 798 1
801
+ 799 1
802
+ 800 1
803
+ 801 1
804
+ 802 1
805
+ 803 1
806
+ 804 0
807
+ 805 1
808
+ 806 1
809
+ 807 1
810
+ 808 1
811
+ 809 0
812
+ 810 1
813
+ 811 1
814
+ 812 1
815
+ 813 0
816
+ 814 0
817
+ 815 1
818
+ 816 1
819
+ 817 1
820
+ 818 1
821
+ 819 1
822
+ 820 1
823
+ 821 1
824
+ 822 1
825
+ 823 1
826
+ 824 1
827
+ 825 1
828
+ 826 1
829
+ 827 1
830
+ 828 1
831
+ 829 1
832
+ 830 1
833
+ 831 1
834
+ 832 1
835
+ 833 1
836
+ 834 1
837
+ 835 1
838
+ 836 0
839
+ 837 1
840
+ 838 1
841
+ 839 1
842
+ 840 1
843
+ 841 0
844
+ 842 1
845
+ 843 1
846
+ 844 0
847
+ 845 1
848
+ 846 1
849
+ 847 1
850
+ 848 1
851
+ 849 1
852
+ 850 1
853
+ 851 1
854
+ 852 1
855
+ 853 1
856
+ 854 1
857
+ 855 1
858
+ 856 1
859
+ 857 0
860
+ 858 1
861
+ 859 1
862
+ 860 1
863
+ 861 1
864
+ 862 1
865
+ 863 1
866
+ 864 1
867
+ 865 1
868
+ 866 1
869
+ 867 0
870
+ 868 1
871
+ 869 1
872
+ 870 1
873
+ 871 1
874
+ 872 1
875
+ 873 1
876
+ 874 1
877
+ 875 1
878
+ 876 0
879
+ 877 1
880
+ 878 0
881
+ 879 1
882
+ 880 1
883
+ 881 1
884
+ 882 0
885
+ 883 1
886
+ 884 1
887
+ 885 1
888
+ 886 1
889
+ 887 0
890
+ 888 0
891
+ 889 1
892
+ 890 1
893
+ 891 1
894
+ 892 1
895
+ 893 1
896
+ 894 1
897
+ 895 1
898
+ 896 1
899
+ 897 1
900
+ 898 1
901
+ 899 1
902
+ 900 1
903
+ 901 1
904
+ 902 1
905
+ 903 1
906
+ 904 1
907
+ 905 1
908
+ 906 1
909
+ 907 1
910
+ 908 1
911
+ 909 1
912
+ 910 0
913
+ 911 1
914
+ 912 1
915
+ 913 1
916
+ 914 0
917
+ 915 0
918
+ 916 1
919
+ 917 1
920
+ 918 1
921
+ 919 1
922
+ 920 1
923
+ 921 1
924
+ 922 1
925
+ 923 1
926
+ 924 1
927
+ 925 1
928
+ 926 1
929
+ 927 1
930
+ 928 1
931
+ 929 1
932
+ 930 1
933
+ 931 1
934
+ 932 1
935
+ 933 1
936
+ 934 1
937
+ 935 1
938
+ 936 1
939
+ 937 1
940
+ 938 1
941
+ 939 1
942
+ 940 0
943
+ 941 1
944
+ 942 1
945
+ 943 1
946
+ 944 1
947
+ 945 1
948
+ 946 1
949
+ 947 1
950
+ 948 1
951
+ 949 1
952
+ 950 1
953
+ 951 1
954
+ 952 1
955
+ 953 1
956
+ 954 1
957
+ 955 1
958
+ 956 0
959
+ 957 1
960
+ 958 1
961
+ 959 1
962
+ 960 1
963
+ 961 0
964
+ 962 1
965
+ 963 1
966
+ 964 1
967
+ 965 1
968
+ 966 1
969
+ 967 1
970
+ 968 1
971
+ 969 1
972
+ 970 1
973
+ 971 0
974
+ 972 1
975
+ 973 1
976
+ 974 1
977
+ 975 1
978
+ 976 1
979
+ 977 1
980
+ 978 0
981
+ 979 1
982
+ 980 1
983
+ 981 1
984
+ 982 1
985
+ 983 1
986
+ 984 1
987
+ 985 1
988
+ 986 1
989
+ 987 1
990
+ 988 1
991
+ 989 1
992
+ 990 1
993
+ 991 1
994
+ 992 0
995
+ 993 1
996
+ 994 1
997
+ 995 0
998
+ 996 1
999
+ 997 1
1000
+ 998 0
1001
+ 999 1
1002
+ 1000 1
1003
+ 1001 0
1004
+ 1002 0
1005
+ 1003 0
1006
+ 1004 1
1007
+ 1005 1
1008
+ 1006 1
1009
+ 1007 1
1010
+ 1008 1
1011
+ 1009 1
1012
+ 1010 1
1013
+ 1011 1
1014
+ 1012 0
1015
+ 1013 0
1016
+ 1014 1
1017
+ 1015 1
1018
+ 1016 1
1019
+ 1017 1
1020
+ 1018 1
finetune/cola/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9b39e6b90ed088a0fa77b30c824334b9235f24848cc3b443a2f539878eaf63e
3
+ size 534594825
finetune/cola/special_tokens_map.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "mask_token": {
6
+ "content": "<mask>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "pad_token": "<pad>",
13
+ "sep_token": "</s>",
14
+ "unk_token": "<unk>"
15
+ }
finetune/cola/tokenizer_config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": {
4
+ "__type": "AddedToken",
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ },
11
+ "cls_token": {
12
+ "__type": "AddedToken",
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "errors": "replace",
28
+ "mask_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<mask>",
31
+ "lstrip": true,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false
35
+ },
36
+ "model_max_length": 512,
37
+ "name_or_path": "final_models/glue_models/structroberta_s2_final/",
38
+ "pad_token": {
39
+ "__type": "AddedToken",
40
+ "content": "<pad>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": false
45
+ },
46
+ "sep_token": {
47
+ "__type": "AddedToken",
48
+ "content": "</s>",
49
+ "lstrip": false,
50
+ "normalized": true,
51
+ "rstrip": false,
52
+ "single_word": false
53
+ },
54
+ "special_tokens_map_file": null,
55
+ "tokenizer_class": "RobertaTokenizer",
56
+ "trim_offsets": true,
57
+ "unk_token": {
58
+ "__type": "AddedToken",
59
+ "content": "<unk>",
60
+ "lstrip": false,
61
+ "normalized": true,
62
+ "rstrip": false,
63
+ "single_word": false
64
+ }
65
+ }
finetune/cola/train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "train_loss": 0.24862313063248345,
4
+ "train_runtime": 353.5006,
5
+ "train_samples": 8164,
6
+ "train_samples_per_second": 230.947,
7
+ "train_steps_per_second": 1.952
8
+ }
finetune/cola/trainer_state.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.8002619515389654,
3
+ "best_model_checkpoint": "final_models/glue_models/structroberta_s2_final//finetune/cola/checkpoint-400",
4
+ "epoch": 10.0,
5
+ "global_step": 690,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 5.8,
12
+ "eval_accuracy": 0.7006869316101074,
13
+ "eval_f1": 0.8002619515389654,
14
+ "eval_loss": 0.9536450505256653,
15
+ "eval_mcc": 0.22514041055870862,
16
+ "eval_runtime": 1.9857,
17
+ "eval_samples_per_second": 513.163,
18
+ "eval_steps_per_second": 64.46,
19
+ "step": 400
20
+ },
21
+ {
22
+ "epoch": 7.25,
23
+ "learning_rate": 1.3768115942028985e-05,
24
+ "loss": 0.3207,
25
+ "step": 500
26
+ },
27
+ {
28
+ "epoch": 10.0,
29
+ "step": 690,
30
+ "total_flos": 6812779740549120.0,
31
+ "train_loss": 0.24862313063248345,
32
+ "train_runtime": 353.5006,
33
+ "train_samples_per_second": 230.947,
34
+ "train_steps_per_second": 1.952
35
+ }
36
+ ],
37
+ "max_steps": 690,
38
+ "num_train_epochs": 10,
39
+ "total_flos": 6812779740549120.0,
40
+ "trial_name": null,
41
+ "trial_params": null
42
+ }
finetune/cola/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b7c40dcdd954d400722d8fc6281e925ed80f9af286c6dfe81e4143b12a49940a
3
+ size 3503
finetune/cola/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
finetune/control_raising_control/all_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.880810022354126,
4
+ "eval_f1": 0.889028038683643,
5
+ "eval_loss": 0.8526787161827087,
6
+ "eval_mcc": 0.7731937974254972,
7
+ "eval_runtime": 26.0435,
8
+ "eval_samples": 13382,
9
+ "eval_samples_per_second": 513.833,
10
+ "eval_steps_per_second": 64.239,
11
+ "train_loss": 0.02885379483330656,
12
+ "train_runtime": 310.6244,
13
+ "train_samples": 6570,
14
+ "train_samples_per_second": 211.509,
15
+ "train_steps_per_second": 1.771
16
+ }
finetune/control_raising_control/config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "final_models/glue_models/structroberta_s2_final/",
3
+ "architectures": [
4
+ "StructRobertaForSequenceClassification"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "auto_map": {
8
+ "AutoConfig": "modeling_structroberta.StructRobertaConfig",
9
+ "AutoModelForMaskedLM": "modeling_structroberta.StructRoberta",
10
+ "AutoModelForSequenceClassification": "modeling_structroberta.StructRobertaForSequenceClassification"
11
+ },
12
+ "bos_token_id": 0,
13
+ "classifier_dropout": null,
14
+ "conv_size": 9,
15
+ "eos_token_id": 2,
16
+ "hidden_act": "gelu",
17
+ "hidden_dropout_prob": 0.1,
18
+ "hidden_size": 768,
19
+ "id2label": {
20
+ "0": 0,
21
+ "1": 1
22
+ },
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 3072,
25
+ "label2id": {
26
+ "0": 0,
27
+ "1": 1
28
+ },
29
+ "layer_norm_eps": 1e-05,
30
+ "max_position_embeddings": 514,
31
+ "model_type": "roberta",
32
+ "n_cntxt_layers": 4,
33
+ "n_cntxt_layers_2": 0,
34
+ "n_parser_layers": 4,
35
+ "num_attention_heads": 12,
36
+ "num_hidden_layers": 8,
37
+ "pad_token_id": 1,
38
+ "position_embedding_type": "absolute",
39
+ "problem_type": "single_label_classification",
40
+ "relations": [
41
+ "head",
42
+ "child"
43
+ ],
44
+ "torch_dtype": "float32",
45
+ "transformers_version": "4.26.1",
46
+ "type_vocab_size": 1,
47
+ "use_cache": true,
48
+ "vocab_size": 32000,
49
+ "weight_act": "softmax"
50
+ }
finetune/control_raising_control/eval_results.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.880810022354126,
4
+ "eval_f1": 0.889028038683643,
5
+ "eval_loss": 0.8526787161827087,
6
+ "eval_mcc": 0.7731937974254972,
7
+ "eval_runtime": 26.0435,
8
+ "eval_samples": 13382,
9
+ "eval_samples_per_second": 513.833,
10
+ "eval_steps_per_second": 64.239
11
+ }
finetune/control_raising_control/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
finetune/control_raising_control/modeling_structroberta.py ADDED
@@ -0,0 +1,2146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import math
3
+ from typing import List, Optional, Tuple, Union
4
+ import torch
5
+ import torch.utils.checkpoint
6
+ from packaging import version
7
+ from torch import nn
8
+ import torch.nn.functional as F
9
+ from torch.nn import CrossEntropyLoss, MSELoss, BCEWithLogitsLoss
10
+ from transformers.activations import ACT2FN, gelu
11
+ from transformers.modeling_outputs import (
12
+ BaseModelOutputWithPastAndCrossAttentions,
13
+ BaseModelOutputWithPoolingAndCrossAttentions,
14
+ MaskedLMOutput,
15
+ SequenceClassifierOutput
16
+ )
17
+ from transformers.modeling_utils import (
18
+ PreTrainedModel,
19
+ apply_chunking_to_forward,
20
+ find_pruneable_heads_and_indices,
21
+ prune_linear_layer,
22
+ )
23
+ from transformers.utils import logging
24
+ from transformers import RobertaConfig
25
+
26
+
27
+ logger = logging.get_logger(__name__)
28
+ ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [
29
+ "roberta-base",
30
+ "roberta-large",
31
+ "roberta-large-mnli",
32
+ "distilroberta-base",
33
+ "roberta-base-openai-detector",
34
+ "roberta-large-openai-detector",
35
+ # See all RoBERTa models at https://huggingface.co/models?filter=roberta
36
+ ]
37
+
38
+
39
+ class StructRobertaConfig(RobertaConfig):
40
+ model_type = "roberta"
41
+
42
+ def __init__(
43
+ self,
44
+ n_parser_layers=4,
45
+ conv_size=9,
46
+ relations=("head", "child"),
47
+ weight_act="softmax",
48
+ n_cntxt_layers=3,
49
+ n_cntxt_layers_2=0,
50
+ **kwargs,):
51
+
52
+ super().__init__(**kwargs)
53
+ self.n_cntxt_layers = n_cntxt_layers
54
+ self.n_parser_layers = n_parser_layers
55
+ self.n_cntxt_layers_2 = n_cntxt_layers_2
56
+ self.conv_size = conv_size
57
+ self.relations = relations
58
+ self.weight_act = weight_act
59
+
60
+ class Conv1d(nn.Module):
61
+ """1D convolution layer."""
62
+
63
+ def __init__(self, hidden_size, kernel_size, dilation=1):
64
+ """Initialization.
65
+
66
+ Args:
67
+ hidden_size: dimension of input embeddings
68
+ kernel_size: convolution kernel size
69
+ dilation: the spacing between the kernel points
70
+ """
71
+ super(Conv1d, self).__init__()
72
+
73
+ if kernel_size % 2 == 0:
74
+ padding = (kernel_size // 2) * dilation
75
+ self.shift = True
76
+ else:
77
+ padding = ((kernel_size - 1) // 2) * dilation
78
+ self.shift = False
79
+ self.conv = nn.Conv1d(
80
+ hidden_size, hidden_size, kernel_size, padding=padding, dilation=dilation
81
+ )
82
+
83
+ def forward(self, x):
84
+ """Compute convolution.
85
+
86
+ Args:
87
+ x: input embeddings
88
+ Returns:
89
+ conv_output: convolution results
90
+ """
91
+
92
+ if self.shift:
93
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)[:, 1:]
94
+ else:
95
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)
96
+
97
+
98
+ class RobertaEmbeddings(nn.Module):
99
+ """
100
+ Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
101
+ """
102
+
103
+ # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__
104
+ def __init__(self, config):
105
+ super().__init__()
106
+ self.word_embeddings = nn.Embedding(
107
+ config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id
108
+ )
109
+ self.position_embeddings = nn.Embedding(
110
+ config.max_position_embeddings, config.hidden_size
111
+ )
112
+ self.token_type_embeddings = nn.Embedding(
113
+ config.type_vocab_size, config.hidden_size
114
+ )
115
+
116
+ # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
117
+ # any TensorFlow checkpoint file
118
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
119
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
120
+ # position_ids (1, len position emb) is contiguous in memory and exported when serialized
121
+ self.position_embedding_type = getattr(
122
+ config, "position_embedding_type", "absolute"
123
+ )
124
+ self.register_buffer(
125
+ "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))
126
+ )
127
+ if version.parse(torch.__version__) > version.parse("1.6.0"):
128
+ self.register_buffer(
129
+ "token_type_ids",
130
+ torch.zeros(self.position_ids.size(), dtype=torch.long),
131
+ persistent=False,
132
+ )
133
+
134
+ # End copy
135
+ self.padding_idx = config.pad_token_id
136
+ self.position_embeddings = nn.Embedding(
137
+ config.max_position_embeddings,
138
+ config.hidden_size,
139
+ padding_idx=self.padding_idx,
140
+ )
141
+
142
+ def forward(
143
+ self,
144
+ input_ids=None,
145
+ token_type_ids=None,
146
+ position_ids=None,
147
+ inputs_embeds=None,
148
+ past_key_values_length=0,
149
+ ):
150
+ if position_ids is None:
151
+ if input_ids is not None:
152
+ # Create the position ids from the input token ids. Any padded tokens remain padded.
153
+ position_ids = create_position_ids_from_input_ids(
154
+ input_ids, self.padding_idx, past_key_values_length
155
+ )
156
+ else:
157
+ position_ids = self.create_position_ids_from_inputs_embeds(
158
+ inputs_embeds
159
+ )
160
+
161
+ if input_ids is not None:
162
+ input_shape = input_ids.size()
163
+ else:
164
+ input_shape = inputs_embeds.size()[:-1]
165
+
166
+ seq_length = input_shape[1]
167
+
168
+ # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs
169
+ # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
170
+ # issue #5664
171
+ if token_type_ids is None:
172
+ if hasattr(self, "token_type_ids"):
173
+ buffered_token_type_ids = self.token_type_ids[:, :seq_length]
174
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
175
+ input_shape[0], seq_length
176
+ )
177
+ token_type_ids = buffered_token_type_ids_expanded
178
+ else:
179
+ token_type_ids = torch.zeros(
180
+ input_shape, dtype=torch.long, device=self.position_ids.device
181
+ )
182
+
183
+ if inputs_embeds is None:
184
+ inputs_embeds = self.word_embeddings(input_ids)
185
+ token_type_embeddings = self.token_type_embeddings(token_type_ids)
186
+
187
+ embeddings = inputs_embeds + token_type_embeddings
188
+ if self.position_embedding_type == "absolute":
189
+ position_embeddings = self.position_embeddings(position_ids)
190
+ embeddings += position_embeddings
191
+ embeddings = self.LayerNorm(embeddings)
192
+ embeddings = self.dropout(embeddings)
193
+ return embeddings
194
+
195
+ def create_position_ids_from_inputs_embeds(self, inputs_embeds):
196
+ """
197
+ We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.
198
+
199
+ Args:
200
+ inputs_embeds: torch.Tensor
201
+
202
+ Returns: torch.Tensor
203
+ """
204
+ input_shape = inputs_embeds.size()[:-1]
205
+ sequence_length = input_shape[1]
206
+
207
+ position_ids = torch.arange(
208
+ self.padding_idx + 1,
209
+ sequence_length + self.padding_idx + 1,
210
+ dtype=torch.long,
211
+ device=inputs_embeds.device,
212
+ )
213
+ return position_ids.unsqueeze(0).expand(input_shape)
214
+
215
+
216
+ # Copied from transformers.models.bert.modeling_bert.BertSelfAttention with Bert->Roberta
217
+ class RobertaSelfAttention(nn.Module):
218
+ def __init__(self, config, position_embedding_type=None):
219
+ super().__init__()
220
+ if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
221
+ config, "embedding_size"
222
+ ):
223
+ raise ValueError(
224
+ f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
225
+ f"heads ({config.num_attention_heads})"
226
+ )
227
+
228
+ self.num_attention_heads = config.num_attention_heads
229
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
230
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
231
+
232
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
233
+ self.key = nn.Linear(config.hidden_size, self.all_head_size)
234
+ self.value = nn.Linear(config.hidden_size, self.all_head_size)
235
+
236
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
237
+ self.position_embedding_type = position_embedding_type or getattr(
238
+ config, "position_embedding_type", "absolute"
239
+ )
240
+ if (
241
+ self.position_embedding_type == "relative_key"
242
+ or self.position_embedding_type == "relative_key_query"
243
+ ):
244
+ self.max_position_embeddings = config.max_position_embeddings
245
+ self.distance_embedding = nn.Embedding(
246
+ 2 * config.max_position_embeddings - 1, self.attention_head_size
247
+ )
248
+
249
+ self.is_decoder = config.is_decoder
250
+
251
+ def transpose_for_scores(self, x):
252
+ new_x_shape = x.size()[:-1] + (
253
+ self.num_attention_heads,
254
+ self.attention_head_size,
255
+ )
256
+ x = x.view(new_x_shape)
257
+ return x.permute(0, 2, 1, 3)
258
+
259
+ def forward(
260
+ self,
261
+ hidden_states: torch.Tensor,
262
+ attention_mask: Optional[torch.FloatTensor] = None,
263
+ head_mask: Optional[torch.FloatTensor] = None,
264
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
265
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
266
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
267
+ output_attentions: Optional[bool] = False,
268
+ parser_att_mask=None,
269
+ ) -> Tuple[torch.Tensor]:
270
+ mixed_query_layer = self.query(hidden_states)
271
+
272
+ # If this is instantiated as a cross-attention module, the keys
273
+ # and values come from an encoder; the attention mask needs to be
274
+ # such that the encoder's padding tokens are not attended to.
275
+ is_cross_attention = encoder_hidden_states is not None
276
+
277
+ if is_cross_attention and past_key_value is not None:
278
+ # reuse k,v, cross_attentions
279
+ key_layer = past_key_value[0]
280
+ value_layer = past_key_value[1]
281
+ attention_mask = encoder_attention_mask
282
+ elif is_cross_attention:
283
+ key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
284
+ value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
285
+ attention_mask = encoder_attention_mask
286
+ elif past_key_value is not None:
287
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
288
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
289
+ key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
290
+ value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
291
+ else:
292
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
293
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
294
+
295
+ query_layer = self.transpose_for_scores(mixed_query_layer)
296
+
297
+ if self.is_decoder:
298
+ # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
299
+ # Further calls to cross_attention layer can then reuse all cross-attention
300
+ # key/value_states (first "if" case)
301
+ # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
302
+ # all previous decoder key/value_states. Further calls to uni-directional self-attention
303
+ # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
304
+ # if encoder bi-directional self-attention `past_key_value` is always `None`
305
+ past_key_value = (key_layer, value_layer)
306
+
307
+ # Take the dot product between "query" and "key" to get the raw attention scores.
308
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
309
+
310
+ if (
311
+ self.position_embedding_type == "relative_key"
312
+ or self.position_embedding_type == "relative_key_query"
313
+ ):
314
+ seq_length = hidden_states.size()[1]
315
+ position_ids_l = torch.arange(
316
+ seq_length, dtype=torch.long, device=hidden_states.device
317
+ ).view(-1, 1)
318
+ position_ids_r = torch.arange(
319
+ seq_length, dtype=torch.long, device=hidden_states.device
320
+ ).view(1, -1)
321
+ distance = position_ids_l - position_ids_r
322
+ positional_embedding = self.distance_embedding(
323
+ distance + self.max_position_embeddings - 1
324
+ )
325
+ positional_embedding = positional_embedding.to(
326
+ dtype=query_layer.dtype
327
+ ) # fp16 compatibility
328
+
329
+ if self.position_embedding_type == "relative_key":
330
+ relative_position_scores = torch.einsum(
331
+ "bhld,lrd->bhlr", query_layer, positional_embedding
332
+ )
333
+ attention_scores = attention_scores + relative_position_scores
334
+ elif self.position_embedding_type == "relative_key_query":
335
+ relative_position_scores_query = torch.einsum(
336
+ "bhld,lrd->bhlr", query_layer, positional_embedding
337
+ )
338
+ relative_position_scores_key = torch.einsum(
339
+ "bhrd,lrd->bhlr", key_layer, positional_embedding
340
+ )
341
+ attention_scores = (
342
+ attention_scores
343
+ + relative_position_scores_query
344
+ + relative_position_scores_key
345
+ )
346
+
347
+ attention_scores = attention_scores / math.sqrt(self.attention_head_size)
348
+ if attention_mask is not None:
349
+ # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
350
+ attention_scores = attention_scores + attention_mask
351
+
352
+ if parser_att_mask is None:
353
+ # Normalize the attention scores to probabilities.
354
+ attention_probs = nn.functional.softmax(attention_scores, dim=-1)
355
+ else:
356
+ attention_probs = torch.sigmoid(attention_scores) * parser_att_mask
357
+
358
+ # This is actually dropping out entire tokens to attend to, which might
359
+ # seem a bit unusual, but is taken from the original Transformer paper.
360
+ attention_probs = self.dropout(attention_probs)
361
+
362
+ # Mask heads if we want to
363
+ if head_mask is not None:
364
+ attention_probs = attention_probs * head_mask
365
+
366
+ context_layer = torch.matmul(attention_probs, value_layer)
367
+
368
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
369
+ new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
370
+ context_layer = context_layer.view(new_context_layer_shape)
371
+
372
+ outputs = (
373
+ (context_layer, attention_probs) if output_attentions else (context_layer,)
374
+ )
375
+
376
+ if self.is_decoder:
377
+ outputs = outputs + (past_key_value,)
378
+ return outputs
379
+
380
+
381
+ # Copied from transformers.models.bert.modeling_bert.BertSelfOutput
382
+ class RobertaSelfOutput(nn.Module):
383
+ def __init__(self, config):
384
+ super().__init__()
385
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
386
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
387
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
388
+
389
+ def forward(
390
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
391
+ ) -> torch.Tensor:
392
+ hidden_states = self.dense(hidden_states)
393
+ hidden_states = self.dropout(hidden_states)
394
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
395
+ return hidden_states
396
+
397
+
398
+ # Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Roberta
399
+ class RobertaAttention(nn.Module):
400
+ def __init__(self, config, position_embedding_type=None):
401
+ super().__init__()
402
+ self.self = RobertaSelfAttention(
403
+ config, position_embedding_type=position_embedding_type
404
+ )
405
+ self.output = RobertaSelfOutput(config)
406
+ self.pruned_heads = set()
407
+
408
+ def prune_heads(self, heads):
409
+ if len(heads) == 0:
410
+ return
411
+ heads, index = find_pruneable_heads_and_indices(
412
+ heads,
413
+ self.self.num_attention_heads,
414
+ self.self.attention_head_size,
415
+ self.pruned_heads,
416
+ )
417
+
418
+ # Prune linear layers
419
+ self.self.query = prune_linear_layer(self.self.query, index)
420
+ self.self.key = prune_linear_layer(self.self.key, index)
421
+ self.self.value = prune_linear_layer(self.self.value, index)
422
+ self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
423
+
424
+ # Update hyper params and store pruned heads
425
+ self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
426
+ self.self.all_head_size = (
427
+ self.self.attention_head_size * self.self.num_attention_heads
428
+ )
429
+ self.pruned_heads = self.pruned_heads.union(heads)
430
+
431
+ def forward(
432
+ self,
433
+ hidden_states: torch.Tensor,
434
+ attention_mask: Optional[torch.FloatTensor] = None,
435
+ head_mask: Optional[torch.FloatTensor] = None,
436
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
437
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
438
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
439
+ output_attentions: Optional[bool] = False,
440
+ parser_att_mask=None,
441
+ ) -> Tuple[torch.Tensor]:
442
+ self_outputs = self.self(
443
+ hidden_states,
444
+ attention_mask,
445
+ head_mask,
446
+ encoder_hidden_states,
447
+ encoder_attention_mask,
448
+ past_key_value,
449
+ output_attentions,
450
+ parser_att_mask=parser_att_mask,
451
+ )
452
+ attention_output = self.output(self_outputs[0], hidden_states)
453
+ outputs = (attention_output,) + self_outputs[
454
+ 1:
455
+ ] # add attentions if we output them
456
+ return outputs
457
+
458
+
459
+ # Copied from transformers.models.bert.modeling_bert.BertIntermediate
460
+ class RobertaIntermediate(nn.Module):
461
+ def __init__(self, config):
462
+ super().__init__()
463
+ self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
464
+ if isinstance(config.hidden_act, str):
465
+ self.intermediate_act_fn = ACT2FN[config.hidden_act]
466
+ else:
467
+ self.intermediate_act_fn = config.hidden_act
468
+
469
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
470
+ hidden_states = self.dense(hidden_states)
471
+ hidden_states = self.intermediate_act_fn(hidden_states)
472
+ return hidden_states
473
+
474
+
475
+ # Copied from transformers.models.bert.modeling_bert.BertOutput
476
+ class RobertaOutput(nn.Module):
477
+ def __init__(self, config):
478
+ super().__init__()
479
+ self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
480
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
481
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
482
+
483
+ def forward(
484
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
485
+ ) -> torch.Tensor:
486
+ hidden_states = self.dense(hidden_states)
487
+ hidden_states = self.dropout(hidden_states)
488
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
489
+ return hidden_states
490
+
491
+
492
+ # Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Roberta
493
+ class RobertaLayer(nn.Module):
494
+ def __init__(self, config):
495
+ super().__init__()
496
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
497
+ self.seq_len_dim = 1
498
+ self.attention = RobertaAttention(config)
499
+ self.is_decoder = config.is_decoder
500
+ self.add_cross_attention = config.add_cross_attention
501
+ if self.add_cross_attention:
502
+ if not self.is_decoder:
503
+ raise ValueError(
504
+ f"{self} should be used as a decoder model if cross attention is added"
505
+ )
506
+ self.crossattention = RobertaAttention(
507
+ config, position_embedding_type="absolute"
508
+ )
509
+ self.intermediate = RobertaIntermediate(config)
510
+ self.output = RobertaOutput(config)
511
+
512
+ def forward(
513
+ self,
514
+ hidden_states: torch.Tensor,
515
+ attention_mask: Optional[torch.FloatTensor] = None,
516
+ head_mask: Optional[torch.FloatTensor] = None,
517
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
518
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
519
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
520
+ output_attentions: Optional[bool] = False,
521
+ parser_att_mask=None,
522
+ ) -> Tuple[torch.Tensor]:
523
+ # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
524
+ self_attn_past_key_value = (
525
+ past_key_value[:2] if past_key_value is not None else None
526
+ )
527
+ self_attention_outputs = self.attention(
528
+ hidden_states,
529
+ attention_mask,
530
+ head_mask,
531
+ output_attentions=output_attentions,
532
+ past_key_value=self_attn_past_key_value,
533
+ parser_att_mask=parser_att_mask,
534
+ )
535
+ attention_output = self_attention_outputs[0]
536
+
537
+ # if decoder, the last output is tuple of self-attn cache
538
+ if self.is_decoder:
539
+ outputs = self_attention_outputs[1:-1]
540
+ present_key_value = self_attention_outputs[-1]
541
+ else:
542
+ outputs = self_attention_outputs[
543
+ 1:
544
+ ] # add self attentions if we output attention weights
545
+
546
+ cross_attn_present_key_value = None
547
+ if self.is_decoder and encoder_hidden_states is not None:
548
+ if not hasattr(self, "crossattention"):
549
+ raise ValueError(
550
+ f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`"
551
+ )
552
+
553
+ # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
554
+ cross_attn_past_key_value = (
555
+ past_key_value[-2:] if past_key_value is not None else None
556
+ )
557
+ cross_attention_outputs = self.crossattention(
558
+ attention_output,
559
+ attention_mask,
560
+ head_mask,
561
+ encoder_hidden_states,
562
+ encoder_attention_mask,
563
+ cross_attn_past_key_value,
564
+ output_attentions,
565
+ )
566
+ attention_output = cross_attention_outputs[0]
567
+ outputs = (
568
+ outputs + cross_attention_outputs[1:-1]
569
+ ) # add cross attentions if we output attention weights
570
+
571
+ # add cross-attn cache to positions 3,4 of present_key_value tuple
572
+ cross_attn_present_key_value = cross_attention_outputs[-1]
573
+ present_key_value = present_key_value + cross_attn_present_key_value
574
+
575
+ layer_output = apply_chunking_to_forward(
576
+ self.feed_forward_chunk,
577
+ self.chunk_size_feed_forward,
578
+ self.seq_len_dim,
579
+ attention_output,
580
+ )
581
+ outputs = (layer_output,) + outputs
582
+
583
+ # if decoder, return the attn key/values as the last output
584
+ if self.is_decoder:
585
+ outputs = outputs + (present_key_value,)
586
+
587
+ return outputs
588
+
589
+ def feed_forward_chunk(self, attention_output):
590
+ intermediate_output = self.intermediate(attention_output)
591
+ layer_output = self.output(intermediate_output, attention_output)
592
+ return layer_output
593
+
594
+
595
+ # Copied from transformers.models.bert.modeling_bert.BertEncoder with Bert->Roberta
596
+ class RobertaEncoder(nn.Module):
597
+ def __init__(self, config):
598
+ super().__init__()
599
+ self.config = config
600
+ self.layer = nn.ModuleList(
601
+ [RobertaLayer(config) for _ in range(config.num_hidden_layers)]
602
+ )
603
+ self.gradient_checkpointing = False
604
+
605
+ def forward(
606
+ self,
607
+ hidden_states: torch.Tensor,
608
+ attention_mask: Optional[torch.FloatTensor] = None,
609
+ head_mask: Optional[torch.FloatTensor] = None,
610
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
611
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
612
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
613
+ use_cache: Optional[bool] = None,
614
+ output_attentions: Optional[bool] = False,
615
+ output_hidden_states: Optional[bool] = False,
616
+ return_dict: Optional[bool] = True,
617
+ parser_att_mask=None,
618
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]:
619
+ all_hidden_states = () if output_hidden_states else None
620
+ all_self_attentions = () if output_attentions else None
621
+ all_cross_attentions = (
622
+ () if output_attentions and self.config.add_cross_attention else None
623
+ )
624
+
625
+ next_decoder_cache = () if use_cache else None
626
+ for i, layer_module in enumerate(self.layer):
627
+ if output_hidden_states:
628
+ all_hidden_states = all_hidden_states + (hidden_states,)
629
+
630
+ layer_head_mask = head_mask[i] if head_mask is not None else None
631
+ past_key_value = past_key_values[i] if past_key_values is not None else None
632
+
633
+ if self.gradient_checkpointing and self.training:
634
+
635
+ if use_cache:
636
+ logger.warning(
637
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
638
+ )
639
+ use_cache = False
640
+
641
+ def create_custom_forward(module):
642
+ def custom_forward(*inputs):
643
+ return module(*inputs, past_key_value, output_attentions)
644
+
645
+ return custom_forward
646
+
647
+ layer_outputs = torch.utils.checkpoint.checkpoint(
648
+ create_custom_forward(layer_module),
649
+ hidden_states,
650
+ attention_mask,
651
+ layer_head_mask,
652
+ encoder_hidden_states,
653
+ encoder_attention_mask,
654
+ )
655
+ else:
656
+ if parser_att_mask is not None:
657
+ layer_outputs = layer_module(
658
+ hidden_states,
659
+ attention_mask,
660
+ layer_head_mask,
661
+ encoder_hidden_states,
662
+ encoder_attention_mask,
663
+ past_key_value,
664
+ output_attentions,
665
+ parser_att_mask=parser_att_mask[i])
666
+ else:
667
+ layer_outputs = layer_module(
668
+ hidden_states,
669
+ attention_mask,
670
+ layer_head_mask,
671
+ encoder_hidden_states,
672
+ encoder_attention_mask,
673
+ past_key_value,
674
+ output_attentions,
675
+ parser_att_mask=None)
676
+
677
+
678
+ hidden_states = layer_outputs[0]
679
+ if use_cache:
680
+ next_decoder_cache += (layer_outputs[-1],)
681
+ if output_attentions:
682
+ all_self_attentions = all_self_attentions + (layer_outputs[1],)
683
+ if self.config.add_cross_attention:
684
+ all_cross_attentions = all_cross_attentions + (layer_outputs[2],)
685
+
686
+ if output_hidden_states:
687
+ all_hidden_states = all_hidden_states + (hidden_states,)
688
+
689
+ if not return_dict:
690
+ return tuple(
691
+ v
692
+ for v in [
693
+ hidden_states,
694
+ next_decoder_cache,
695
+ all_hidden_states,
696
+ all_self_attentions,
697
+ all_cross_attentions,
698
+ ]
699
+ if v is not None
700
+ )
701
+ return BaseModelOutputWithPastAndCrossAttentions(
702
+ last_hidden_state=hidden_states,
703
+ past_key_values=next_decoder_cache,
704
+ hidden_states=all_hidden_states,
705
+ attentions=all_self_attentions,
706
+ cross_attentions=all_cross_attentions,
707
+ )
708
+
709
+
710
+ # Copied from transformers.models.bert.modeling_bert.BertPooler
711
+ class RobertaPooler(nn.Module):
712
+ def __init__(self, config):
713
+ super().__init__()
714
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
715
+ self.activation = nn.Tanh()
716
+
717
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
718
+ # We "pool" the model by simply taking the hidden state corresponding
719
+ # to the first token.
720
+ first_token_tensor = hidden_states[:, 0]
721
+ pooled_output = self.dense(first_token_tensor)
722
+ pooled_output = self.activation(pooled_output)
723
+ return pooled_output
724
+
725
+
726
+ class RobertaPreTrainedModel(PreTrainedModel):
727
+ """
728
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
729
+ models.
730
+ """
731
+
732
+ config_class = RobertaConfig
733
+ base_model_prefix = "roberta"
734
+ supports_gradient_checkpointing = True
735
+
736
+ # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights
737
+ def _init_weights(self, module):
738
+ """Initialize the weights"""
739
+ if isinstance(module, nn.Linear):
740
+ # Slightly different from the TF version which uses truncated_normal for initialization
741
+ # cf https://github.com/pytorch/pytorch/pull/5617
742
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
743
+ if module.bias is not None:
744
+ module.bias.data.zero_()
745
+ elif isinstance(module, nn.Embedding):
746
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
747
+ if module.padding_idx is not None:
748
+ module.weight.data[module.padding_idx].zero_()
749
+ elif isinstance(module, nn.LayerNorm):
750
+ if module.bias is not None:
751
+ module.bias.data.zero_()
752
+ module.weight.data.fill_(1.0)
753
+
754
+ def _set_gradient_checkpointing(self, module, value=False):
755
+ if isinstance(module, RobertaEncoder):
756
+ module.gradient_checkpointing = value
757
+
758
+ def update_keys_to_ignore(self, config, del_keys_to_ignore):
759
+ """Remove some keys from ignore list"""
760
+ if not config.tie_word_embeddings:
761
+ # must make a new list, or the class variable gets modified!
762
+ self._keys_to_ignore_on_save = [
763
+ k for k in self._keys_to_ignore_on_save if k not in del_keys_to_ignore
764
+ ]
765
+ self._keys_to_ignore_on_load_missing = [
766
+ k
767
+ for k in self._keys_to_ignore_on_load_missing
768
+ if k not in del_keys_to_ignore
769
+ ]
770
+
771
+
772
+ ROBERTA_START_DOCSTRING = r"""
773
+
774
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
775
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
776
+ etc.)
777
+
778
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
779
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
780
+ and behavior.
781
+
782
+ Parameters:
783
+ config ([`RobertaConfig`]): Model configuration class with all the parameters of the
784
+ model. Initializing with a config file does not load the weights associated with the model, only the
785
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
786
+ """
787
+
788
+
789
+ ROBERTA_INPUTS_DOCSTRING = r"""
790
+ Args:
791
+ input_ids (`torch.LongTensor` of shape `({0})`):
792
+ Indices of input sequence tokens in the vocabulary.
793
+
794
+ Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
795
+ [`PreTrainedTokenizer.__call__`] for details.
796
+
797
+ [What are input IDs?](../glossary#input-ids)
798
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
799
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
800
+
801
+ - 1 for tokens that are **not masked**,
802
+ - 0 for tokens that are **masked**.
803
+
804
+ [What are attention masks?](../glossary#attention-mask)
805
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
806
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
807
+ 1]`:
808
+
809
+ - 0 corresponds to a *sentence A* token,
810
+ - 1 corresponds to a *sentence B* token.
811
+
812
+ [What are token type IDs?](../glossary#token-type-ids)
813
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
814
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
815
+ config.max_position_embeddings - 1]`.
816
+
817
+ [What are position IDs?](../glossary#position-ids)
818
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
819
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
820
+
821
+ - 1 indicates the head is **not masked**,
822
+ - 0 indicates the head is **masked**.
823
+
824
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
825
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
826
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
827
+ model's internal embedding lookup matrix.
828
+ output_attentions (`bool`, *optional*):
829
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
830
+ tensors for more detail.
831
+ output_hidden_states (`bool`, *optional*):
832
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
833
+ more detail.
834
+ return_dict (`bool`, *optional*):
835
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
836
+ """
837
+
838
+
839
+ class RobertaModel(RobertaPreTrainedModel):
840
+ """
841
+
842
+ The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
843
+ cross-attention is added between the self-attention layers, following the architecture described in *Attention is
844
+ all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
845
+ Kaiser and Illia Polosukhin.
846
+
847
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
848
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
849
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
850
+
851
+ .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762
852
+
853
+ """
854
+
855
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
856
+
857
+ # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->Roberta
858
+ def __init__(self, config, add_pooling_layer=True):
859
+ super().__init__(config)
860
+ self.config = config
861
+
862
+ self.embeddings = RobertaEmbeddings(config)
863
+ self.encoder = RobertaEncoder(config)
864
+
865
+ self.pooler = RobertaPooler(config) if add_pooling_layer else None
866
+
867
+ # Initialize weights and apply final processing
868
+ self.post_init()
869
+
870
+ def get_input_embeddings(self):
871
+ return self.embeddings.word_embeddings
872
+
873
+ def set_input_embeddings(self, value):
874
+ self.embeddings.word_embeddings = value
875
+
876
+ def _prune_heads(self, heads_to_prune):
877
+ """
878
+ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
879
+ class PreTrainedModel
880
+ """
881
+ for layer, heads in heads_to_prune.items():
882
+ self.encoder.layer[layer].attention.prune_heads(heads)
883
+
884
+ # Copied from transformers.models.bert.modeling_bert.BertModel.forward
885
+ def forward(
886
+ self,
887
+ input_ids: Optional[torch.Tensor] = None,
888
+ attention_mask: Optional[torch.Tensor] = None,
889
+ token_type_ids: Optional[torch.Tensor] = None,
890
+ position_ids: Optional[torch.Tensor] = None,
891
+ head_mask: Optional[torch.Tensor] = None,
892
+ inputs_embeds: Optional[torch.Tensor] = None,
893
+ encoder_hidden_states: Optional[torch.Tensor] = None,
894
+ encoder_attention_mask: Optional[torch.Tensor] = None,
895
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
896
+ use_cache: Optional[bool] = None,
897
+ output_attentions: Optional[bool] = None,
898
+ output_hidden_states: Optional[bool] = None,
899
+ return_dict: Optional[bool] = None,
900
+ parser_att_mask=None,
901
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]:
902
+ r"""
903
+ encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
904
+ Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
905
+ the model is configured as a decoder.
906
+ encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
907
+ Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
908
+ the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:
909
+
910
+ - 1 for tokens that are **not masked**,
911
+ - 0 for tokens that are **masked**.
912
+ past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
913
+ Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
914
+
915
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
916
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
917
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
918
+ use_cache (`bool`, *optional*):
919
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
920
+ `past_key_values`).
921
+ """
922
+ output_attentions = (
923
+ output_attentions
924
+ if output_attentions is not None
925
+ else self.config.output_attentions
926
+ )
927
+ output_hidden_states = (
928
+ output_hidden_states
929
+ if output_hidden_states is not None
930
+ else self.config.output_hidden_states
931
+ )
932
+ return_dict = (
933
+ return_dict if return_dict is not None else self.config.use_return_dict
934
+ )
935
+
936
+ if self.config.is_decoder:
937
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
938
+ else:
939
+ use_cache = False
940
+
941
+ if input_ids is not None and inputs_embeds is not None:
942
+ raise ValueError(
943
+ "You cannot specify both input_ids and inputs_embeds at the same time"
944
+ )
945
+ elif input_ids is not None:
946
+ input_shape = input_ids.size()
947
+ elif inputs_embeds is not None:
948
+ input_shape = inputs_embeds.size()[:-1]
949
+ else:
950
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
951
+
952
+ batch_size, seq_length = input_shape
953
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
954
+
955
+ # past_key_values_length
956
+ past_key_values_length = (
957
+ past_key_values[0][0].shape[2] if past_key_values is not None else 0
958
+ )
959
+
960
+ if attention_mask is None:
961
+ attention_mask = torch.ones(
962
+ ((batch_size, seq_length + past_key_values_length)), device=device
963
+ )
964
+
965
+ if token_type_ids is None:
966
+ if hasattr(self.embeddings, "token_type_ids"):
967
+ buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
968
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
969
+ batch_size, seq_length
970
+ )
971
+ token_type_ids = buffered_token_type_ids_expanded
972
+ else:
973
+ token_type_ids = torch.zeros(
974
+ input_shape, dtype=torch.long, device=device
975
+ )
976
+
977
+ # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
978
+ # ourselves in which case we just need to make it broadcastable to all heads.
979
+ extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(
980
+ attention_mask, input_shape, device
981
+ )
982
+
983
+ # If a 2D or 3D attention mask is provided for the cross-attention
984
+ # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
985
+ if self.config.is_decoder and encoder_hidden_states is not None:
986
+ (
987
+ encoder_batch_size,
988
+ encoder_sequence_length,
989
+ _,
990
+ ) = encoder_hidden_states.size()
991
+ encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
992
+ if encoder_attention_mask is None:
993
+ encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
994
+ encoder_extended_attention_mask = self.invert_attention_mask(
995
+ encoder_attention_mask
996
+ )
997
+ else:
998
+ encoder_extended_attention_mask = None
999
+
1000
+ # Prepare head mask if needed
1001
+ # 1.0 in head_mask indicate we keep the head
1002
+ # attention_probs has shape bsz x n_heads x N x N
1003
+ # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
1004
+ # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
1005
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
1006
+
1007
+ embedding_output = self.embeddings(
1008
+ input_ids=input_ids,
1009
+ position_ids=position_ids,
1010
+ token_type_ids=token_type_ids,
1011
+ inputs_embeds=inputs_embeds,
1012
+ past_key_values_length=past_key_values_length,
1013
+ )
1014
+ encoder_outputs = self.encoder(
1015
+ embedding_output,
1016
+ attention_mask=extended_attention_mask,
1017
+ head_mask=head_mask,
1018
+ encoder_hidden_states=encoder_hidden_states,
1019
+ encoder_attention_mask=encoder_extended_attention_mask,
1020
+ past_key_values=past_key_values,
1021
+ use_cache=use_cache,
1022
+ output_attentions=output_attentions,
1023
+ output_hidden_states=output_hidden_states,
1024
+ return_dict=return_dict,
1025
+ parser_att_mask=parser_att_mask,
1026
+ )
1027
+ sequence_output = encoder_outputs[0]
1028
+ pooled_output = (
1029
+ self.pooler(sequence_output) if self.pooler is not None else None
1030
+ )
1031
+
1032
+ if not return_dict:
1033
+ return (sequence_output, pooled_output) + encoder_outputs[1:]
1034
+
1035
+ return BaseModelOutputWithPoolingAndCrossAttentions(
1036
+ last_hidden_state=sequence_output,
1037
+ pooler_output=pooled_output,
1038
+ past_key_values=encoder_outputs.past_key_values,
1039
+ hidden_states=encoder_outputs.hidden_states,
1040
+ attentions=encoder_outputs.attentions,
1041
+ cross_attentions=encoder_outputs.cross_attentions,
1042
+ )
1043
+
1044
+
1045
+ class StructRoberta(RobertaPreTrainedModel):
1046
+ _keys_to_ignore_on_save = [r"lm_head.decoder.weight", r"lm_head.decoder.bias"]
1047
+ _keys_to_ignore_on_load_missing = [
1048
+ r"position_ids",
1049
+ r"lm_head.decoder.weight",
1050
+ r"lm_head.decoder.bias",
1051
+ ]
1052
+ _keys_to_ignore_on_load_unexpected = [r"pooler"]
1053
+
1054
+ def __init__(self, config):
1055
+ super().__init__(config)
1056
+
1057
+ if config.is_decoder:
1058
+ logger.warning(
1059
+ "If you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for "
1060
+ "bi-directional self-attention."
1061
+ )
1062
+
1063
+
1064
+ if config.n_cntxt_layers > 0:
1065
+ config_cntxt = copy.deepcopy(config)
1066
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1067
+
1068
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1069
+
1070
+ if config.n_cntxt_layers_2 > 0:
1071
+ self.parser_layers_1 = nn.ModuleList(
1072
+ [
1073
+ nn.Sequential(
1074
+ Conv1d(config.hidden_size, config.conv_size),
1075
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1076
+ nn.Tanh(),
1077
+ )
1078
+ for i in range(int(config.n_parser_layers/2))
1079
+ ]
1080
+ )
1081
+
1082
+ self.distance_ff_1 = nn.Sequential(
1083
+ Conv1d(config.hidden_size, 2),
1084
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1085
+ nn.Tanh(),
1086
+ nn.Linear(config.hidden_size, 1),
1087
+ )
1088
+
1089
+ self.height_ff_1 = nn.Sequential(
1090
+ nn.Linear(config.hidden_size, config.hidden_size),
1091
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1092
+ nn.Tanh(),
1093
+ nn.Linear(config.hidden_size, 1),
1094
+ )
1095
+
1096
+ n_rel = len(config.relations)
1097
+ self._rel_weight_1 = nn.Parameter(
1098
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1099
+ )
1100
+ self._rel_weight_1.data.normal_(0, 0.1)
1101
+
1102
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1103
+
1104
+ config_cntxt_2 = copy.deepcopy(config)
1105
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1106
+
1107
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1108
+
1109
+
1110
+ self.parser_layers_2 = nn.ModuleList(
1111
+ [
1112
+ nn.Sequential(
1113
+ Conv1d(config.hidden_size, config.conv_size),
1114
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1115
+ nn.Tanh(),
1116
+ )
1117
+ for i in range(int(config.n_parser_layers/2))
1118
+ ]
1119
+ )
1120
+
1121
+ self.distance_ff_2 = nn.Sequential(
1122
+ Conv1d(config.hidden_size, 2),
1123
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1124
+ nn.Tanh(),
1125
+ nn.Linear(config.hidden_size, 1),
1126
+ )
1127
+
1128
+ self.height_ff_2 = nn.Sequential(
1129
+ nn.Linear(config.hidden_size, config.hidden_size),
1130
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1131
+ nn.Tanh(),
1132
+ nn.Linear(config.hidden_size, 1),
1133
+ )
1134
+
1135
+ n_rel = len(config.relations)
1136
+ self._rel_weight_2 = nn.Parameter(
1137
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1138
+ )
1139
+ self._rel_weight_2.data.normal_(0, 0.1)
1140
+
1141
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1142
+
1143
+ else:
1144
+ self.parser_layers = nn.ModuleList(
1145
+ [
1146
+ nn.Sequential(
1147
+ Conv1d(config.hidden_size, config.conv_size),
1148
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1149
+ nn.Tanh(),
1150
+ )
1151
+ for i in range(config.n_parser_layers)
1152
+ ]
1153
+ )
1154
+
1155
+ self.distance_ff = nn.Sequential(
1156
+ Conv1d(config.hidden_size, 2),
1157
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1158
+ nn.Tanh(),
1159
+ nn.Linear(config.hidden_size, 1),
1160
+ )
1161
+
1162
+ self.height_ff = nn.Sequential(
1163
+ nn.Linear(config.hidden_size, config.hidden_size),
1164
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1165
+ nn.Tanh(),
1166
+ nn.Linear(config.hidden_size, 1),
1167
+ )
1168
+
1169
+ n_rel = len(config.relations)
1170
+ self._rel_weight = nn.Parameter(
1171
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1172
+ )
1173
+ self._rel_weight.data.normal_(0, 0.1)
1174
+
1175
+ self._scaler = nn.Parameter(torch.zeros(2))
1176
+
1177
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1178
+
1179
+ if config.n_cntxt_layers > 0:
1180
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1181
+ if config.n_cntxt_layers_2 > 0:
1182
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1183
+
1184
+ self.lm_head = RobertaLMHead(config)
1185
+
1186
+ self.pad = config.pad_token_id
1187
+
1188
+ # The LM head weights require special treatment only when they are tied with the word embeddings
1189
+ self.update_keys_to_ignore(config, ["lm_head.decoder.weight"])
1190
+
1191
+ # Initialize weights and apply final processing
1192
+ self.post_init()
1193
+
1194
+ def get_output_embeddings(self):
1195
+ return self.lm_head.decoder
1196
+
1197
+ def set_output_embeddings(self, new_embeddings):
1198
+ self.lm_head.decoder = new_embeddings
1199
+
1200
+ @property
1201
+ def scaler(self):
1202
+ return self._scaler.exp()
1203
+
1204
+ @property
1205
+ def scaler_1(self):
1206
+ return self._scaler_1.exp()
1207
+
1208
+ @property
1209
+ def scaler_2(self):
1210
+ return self._scaler_2.exp()
1211
+
1212
+ @property
1213
+ def rel_weight(self):
1214
+ if self.config.weight_act == "sigmoid":
1215
+ return torch.sigmoid(self._rel_weight)
1216
+ elif self.config.weight_act == "softmax":
1217
+ return torch.softmax(self._rel_weight, dim=-1)
1218
+
1219
+ @property
1220
+ def rel_weight_1(self):
1221
+ if self.config.weight_act == "sigmoid":
1222
+ return torch.sigmoid(self._rel_weight_1)
1223
+ elif self.config.weight_act == "softmax":
1224
+ return torch.softmax(self._rel_weight_1, dim=-1)
1225
+
1226
+
1227
+ @property
1228
+ def rel_weight_2(self):
1229
+ if self.config.weight_act == "sigmoid":
1230
+ return torch.sigmoid(self._rel_weight_2)
1231
+ elif self.config.weight_act == "softmax":
1232
+ return torch.softmax(self._rel_weight_2, dim=-1)
1233
+
1234
+
1235
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1236
+ """Compute constituents from distance and height."""
1237
+
1238
+ if n_cntxt_layers>0:
1239
+ if n_cntxt_layers == 1:
1240
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1241
+ elif n_cntxt_layers == 2:
1242
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1243
+ else:
1244
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1245
+
1246
+ gamma = torch.sigmoid(-beta_logits)
1247
+ ones = torch.ones_like(gamma)
1248
+
1249
+ block_mask_left = cummin(
1250
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1251
+ )
1252
+ block_mask_left = block_mask_left - F.pad(
1253
+ block_mask_left[:, :, :-1], (1, 0), value=0
1254
+ )
1255
+ block_mask_left.tril_(0)
1256
+
1257
+ block_mask_right = cummin(
1258
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1259
+ )
1260
+ block_mask_right = block_mask_right - F.pad(
1261
+ block_mask_right[:, :, 1:], (0, 1), value=0
1262
+ )
1263
+ block_mask_right.triu_(0)
1264
+
1265
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1266
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1267
+ block_mask_right, reverse=True
1268
+ ).triu(1)
1269
+
1270
+ return block_p, block
1271
+
1272
+ def compute_head(self, height, n_cntxt_layers=0):
1273
+ """Estimate head for each constituent."""
1274
+
1275
+ _, length = height.size()
1276
+ if n_cntxt_layers>0:
1277
+ if n_cntxt_layers == 1:
1278
+ head_logits = height * self.scaler_1[1]
1279
+ elif n_cntxt_layers == 2:
1280
+ head_logits = height * self.scaler_2[1]
1281
+ else:
1282
+ head_logits = height * self.scaler[1]
1283
+ index = torch.arange(length, device=height.device)
1284
+
1285
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1286
+ index[None, None, :] <= index[None, :, None]
1287
+ )
1288
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1289
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1290
+
1291
+ head_p = torch.softmax(head_logits, dim=-1)
1292
+
1293
+ return head_p
1294
+
1295
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1296
+ """Parse input sentence.
1297
+
1298
+ Args:
1299
+ x: input tokens (required).
1300
+ pos: position for each token (optional).
1301
+ Returns:
1302
+ distance: syntactic distance
1303
+ height: syntactic height
1304
+ """
1305
+
1306
+ mask = x != self.pad
1307
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1308
+
1309
+ if embs is None:
1310
+ h = self.roberta.embeddings(x)
1311
+ else:
1312
+ h = embs
1313
+
1314
+ if n_cntxt_layers > 0:
1315
+ if n_cntxt_layers == 1:
1316
+ parser_layers = self.parser_layers_1
1317
+ height_ff = self.height_ff_1
1318
+ distance_ff = self.distance_ff_1
1319
+ elif n_cntxt_layers == 2:
1320
+ parser_layers = self.parser_layers_2
1321
+ height_ff = self.height_ff_2
1322
+ distance_ff = self.distance_ff_2
1323
+ for i in range(int(self.config.n_parser_layers/2)):
1324
+ h = h.masked_fill(~mask[:, :, None], 0)
1325
+ h = parser_layers[i](h)
1326
+
1327
+ height = height_ff(h).squeeze(-1)
1328
+ height.masked_fill_(~mask, -1e9)
1329
+
1330
+ distance = distance_ff(h).squeeze(-1)
1331
+ distance.masked_fill_(~mask_shifted, 1e9)
1332
+
1333
+ # Calbrating the distance and height to the same level
1334
+ length = distance.size(1)
1335
+ height_max = height[:, None, :].expand(-1, length, -1)
1336
+ height_max = torch.cummax(
1337
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1338
+ )[0].triu(0)
1339
+
1340
+ margin_left = torch.relu(
1341
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1342
+ )
1343
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1344
+ margin = torch.where(
1345
+ margin_left > margin_right, margin_right, margin_left
1346
+ ).triu(0)
1347
+
1348
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1349
+ margin.masked_fill_(~margin_mask, 0)
1350
+ margin = margin.max()
1351
+
1352
+ distance = distance - margin
1353
+ else:
1354
+ for i in range(self.config.n_parser_layers):
1355
+ h = h.masked_fill(~mask[:, :, None], 0)
1356
+ h = self.parser_layers[i](h)
1357
+
1358
+ height = self.height_ff(h).squeeze(-1)
1359
+ height.masked_fill_(~mask, -1e9)
1360
+
1361
+ distance = self.distance_ff(h).squeeze(-1)
1362
+ distance.masked_fill_(~mask_shifted, 1e9)
1363
+
1364
+ # Calbrating the distance and height to the same level
1365
+ length = distance.size(1)
1366
+ height_max = height[:, None, :].expand(-1, length, -1)
1367
+ height_max = torch.cummax(
1368
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1369
+ )[0].triu(0)
1370
+
1371
+ margin_left = torch.relu(
1372
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1373
+ )
1374
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1375
+ margin = torch.where(
1376
+ margin_left > margin_right, margin_right, margin_left
1377
+ ).triu(0)
1378
+
1379
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1380
+ margin.masked_fill_(~margin_mask, 0)
1381
+ margin = margin.max()
1382
+
1383
+ distance = distance - margin
1384
+
1385
+ return distance, height
1386
+
1387
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1388
+ """Compute head and cibling distribution for each token."""
1389
+
1390
+ bsz, length = x.size()
1391
+
1392
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1393
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1394
+
1395
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1396
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1397
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1398
+ head = head.masked_fill(eye, 0)
1399
+ child = head.transpose(1, 2)
1400
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1401
+
1402
+ rel_list = []
1403
+ if "head" in self.config.relations:
1404
+ rel_list.append(head)
1405
+ if "child" in self.config.relations:
1406
+ rel_list.append(child)
1407
+ if "cibling" in self.config.relations:
1408
+ rel_list.append(cibling)
1409
+
1410
+ rel = torch.stack(rel_list, dim=1)
1411
+
1412
+ if n_cntxt_layers > 0:
1413
+ if n_cntxt_layers == 1:
1414
+ rel_weight = self.rel_weight_1
1415
+ elif n_cntxt_layers == 2:
1416
+ rel_weight = self.rel_weight_2
1417
+ else:
1418
+ rel_weight = self.rel_weight
1419
+
1420
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1421
+
1422
+ if n_cntxt_layers == 1:
1423
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1424
+ else:
1425
+ num_layers = self.roberta.config.num_hidden_layers
1426
+
1427
+ att_mask = dep.reshape(
1428
+ num_layers,
1429
+ bsz,
1430
+ self.config.num_attention_heads,
1431
+ length,
1432
+ length,
1433
+ )
1434
+
1435
+ return att_mask, cibling, head, block
1436
+
1437
+ def forward(
1438
+ self,
1439
+ input_ids: Optional[torch.LongTensor] = None,
1440
+ attention_mask: Optional[torch.FloatTensor] = None,
1441
+ token_type_ids: Optional[torch.LongTensor] = None,
1442
+ position_ids: Optional[torch.LongTensor] = None,
1443
+ head_mask: Optional[torch.FloatTensor] = None,
1444
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1445
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
1446
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
1447
+ labels: Optional[torch.LongTensor] = None,
1448
+ output_attentions: Optional[bool] = None,
1449
+ output_hidden_states: Optional[bool] = None,
1450
+ return_dict: Optional[bool] = None,
1451
+ ) -> Union[Tuple, MaskedLMOutput]:
1452
+ r"""
1453
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1454
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
1455
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
1456
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
1457
+ kwargs (`Dict[str, any]`, optional, defaults to *{}*):
1458
+ Used to hide legacy arguments that have been deprecated.
1459
+ """
1460
+ return_dict = (
1461
+ return_dict if return_dict is not None else self.config.use_return_dict
1462
+ )
1463
+
1464
+
1465
+ if self.config.n_cntxt_layers > 0:
1466
+ cntxt_outputs = self.cntxt_layers(
1467
+ input_ids,
1468
+ attention_mask=attention_mask,
1469
+ token_type_ids=token_type_ids,
1470
+ position_ids=position_ids,
1471
+ head_mask=head_mask,
1472
+ inputs_embeds=inputs_embeds,
1473
+ encoder_hidden_states=encoder_hidden_states,
1474
+ encoder_attention_mask=encoder_attention_mask,
1475
+ output_attentions=output_attentions,
1476
+ output_hidden_states=output_hidden_states,
1477
+ return_dict=return_dict)
1478
+
1479
+
1480
+ if self.config.n_cntxt_layers_2 > 0:
1481
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1482
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1483
+
1484
+ cntxt_outputs_2 = self.cntxt_layers_2(
1485
+ input_ids,
1486
+ attention_mask=attention_mask,
1487
+ token_type_ids=token_type_ids,
1488
+ position_ids=position_ids,
1489
+ head_mask=head_mask,
1490
+ inputs_embeds=inputs_embeds,
1491
+ encoder_hidden_states=encoder_hidden_states,
1492
+ encoder_attention_mask=encoder_attention_mask,
1493
+ output_attentions=output_attentions,
1494
+ output_hidden_states=output_hidden_states,
1495
+ return_dict=return_dict,
1496
+ parser_att_mask=att_mask_1)
1497
+
1498
+ sequence_output = cntxt_outputs_2[0]
1499
+
1500
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
1501
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
1502
+
1503
+ elif self.config.n_cntxt_layers > 0:
1504
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
1505
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1506
+ else:
1507
+ distance, height = self.parse(input_ids)
1508
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1509
+
1510
+ outputs = self.roberta(
1511
+ input_ids,
1512
+ attention_mask=attention_mask,
1513
+ token_type_ids=token_type_ids,
1514
+ position_ids=position_ids,
1515
+ head_mask=head_mask,
1516
+ inputs_embeds=inputs_embeds,
1517
+ encoder_hidden_states=encoder_hidden_states,
1518
+ encoder_attention_mask=encoder_attention_mask,
1519
+ output_attentions=output_attentions,
1520
+ output_hidden_states=output_hidden_states,
1521
+ return_dict=return_dict,
1522
+ parser_att_mask=att_mask,
1523
+ )
1524
+ sequence_output = outputs[0]
1525
+ prediction_scores = self.lm_head(sequence_output)
1526
+
1527
+ masked_lm_loss = None
1528
+ if labels is not None:
1529
+ loss_fct = CrossEntropyLoss()
1530
+ masked_lm_loss = loss_fct(
1531
+ prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)
1532
+ )
1533
+
1534
+ if not return_dict:
1535
+ output = (prediction_scores,) + outputs[2:]
1536
+ return (
1537
+ ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
1538
+ )
1539
+
1540
+ return MaskedLMOutput(
1541
+ loss=masked_lm_loss,
1542
+ logits=prediction_scores,
1543
+ hidden_states=outputs.hidden_states,
1544
+ attentions=outputs.attentions,
1545
+ )
1546
+
1547
+
1548
+ class RobertaLMHead(nn.Module):
1549
+ """Roberta Head for masked language modeling."""
1550
+
1551
+ def __init__(self, config):
1552
+ super().__init__()
1553
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
1554
+ self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
1555
+
1556
+ self.decoder = nn.Linear(config.hidden_size, config.vocab_size)
1557
+ self.bias = nn.Parameter(torch.zeros(config.vocab_size))
1558
+ self.decoder.bias = self.bias
1559
+
1560
+ def forward(self, features, **kwargs):
1561
+ x = self.dense(features)
1562
+ x = gelu(x)
1563
+ x = self.layer_norm(x)
1564
+
1565
+ # project back to size of vocabulary with bias
1566
+ x = self.decoder(x)
1567
+
1568
+ return x
1569
+
1570
+ def _tie_weights(self):
1571
+ # To tie those two weights if they get disconnected (on TPU or when the bias is resized)
1572
+ self.bias = self.decoder.bias
1573
+
1574
+
1575
+ class StructRobertaForSequenceClassification(RobertaPreTrainedModel):
1576
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
1577
+
1578
+ def __init__(self, config):
1579
+ super().__init__(config)
1580
+ self.num_labels = config.num_labels
1581
+ self.config = config
1582
+
1583
+ if config.n_cntxt_layers > 0:
1584
+ config_cntxt = copy.deepcopy(config)
1585
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1586
+
1587
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1588
+
1589
+ if config.n_cntxt_layers_2 > 0:
1590
+ self.parser_layers_1 = nn.ModuleList(
1591
+ [
1592
+ nn.Sequential(
1593
+ Conv1d(config.hidden_size, config.conv_size),
1594
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1595
+ nn.Tanh(),
1596
+ )
1597
+ for i in range(int(config.n_parser_layers/2))
1598
+ ]
1599
+ )
1600
+
1601
+ self.distance_ff_1 = nn.Sequential(
1602
+ Conv1d(config.hidden_size, 2),
1603
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1604
+ nn.Tanh(),
1605
+ nn.Linear(config.hidden_size, 1),
1606
+ )
1607
+
1608
+ self.height_ff_1 = nn.Sequential(
1609
+ nn.Linear(config.hidden_size, config.hidden_size),
1610
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1611
+ nn.Tanh(),
1612
+ nn.Linear(config.hidden_size, 1),
1613
+ )
1614
+
1615
+ n_rel = len(config.relations)
1616
+ self._rel_weight_1 = nn.Parameter(
1617
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1618
+ )
1619
+ self._rel_weight_1.data.normal_(0, 0.1)
1620
+
1621
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1622
+
1623
+ config_cntxt_2 = copy.deepcopy(config)
1624
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1625
+
1626
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1627
+
1628
+
1629
+ self.parser_layers_2 = nn.ModuleList(
1630
+ [
1631
+ nn.Sequential(
1632
+ Conv1d(config.hidden_size, config.conv_size),
1633
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1634
+ nn.Tanh(),
1635
+ )
1636
+ for i in range(int(config.n_parser_layers/2))
1637
+ ]
1638
+ )
1639
+
1640
+ self.distance_ff_2 = nn.Sequential(
1641
+ Conv1d(config.hidden_size, 2),
1642
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1643
+ nn.Tanh(),
1644
+ nn.Linear(config.hidden_size, 1),
1645
+ )
1646
+
1647
+ self.height_ff_2 = nn.Sequential(
1648
+ nn.Linear(config.hidden_size, config.hidden_size),
1649
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1650
+ nn.Tanh(),
1651
+ nn.Linear(config.hidden_size, 1),
1652
+ )
1653
+
1654
+ n_rel = len(config.relations)
1655
+ self._rel_weight_2 = nn.Parameter(
1656
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1657
+ )
1658
+ self._rel_weight_2.data.normal_(0, 0.1)
1659
+
1660
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1661
+
1662
+ else:
1663
+ self.parser_layers = nn.ModuleList(
1664
+ [
1665
+ nn.Sequential(
1666
+ Conv1d(config.hidden_size, config.conv_size),
1667
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1668
+ nn.Tanh(),
1669
+ )
1670
+ for i in range(config.n_parser_layers)
1671
+ ]
1672
+ )
1673
+
1674
+ self.distance_ff = nn.Sequential(
1675
+ Conv1d(config.hidden_size, 2),
1676
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1677
+ nn.Tanh(),
1678
+ nn.Linear(config.hidden_size, 1),
1679
+ )
1680
+
1681
+ self.height_ff = nn.Sequential(
1682
+ nn.Linear(config.hidden_size, config.hidden_size),
1683
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1684
+ nn.Tanh(),
1685
+ nn.Linear(config.hidden_size, 1),
1686
+ )
1687
+
1688
+ n_rel = len(config.relations)
1689
+ self._rel_weight = nn.Parameter(
1690
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1691
+ )
1692
+ self._rel_weight.data.normal_(0, 0.1)
1693
+
1694
+ self._scaler = nn.Parameter(torch.zeros(2))
1695
+
1696
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1697
+
1698
+ if config.n_cntxt_layers > 0:
1699
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1700
+ if config.n_cntxt_layers_2 > 0:
1701
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1702
+
1703
+
1704
+ self.pad = config.pad_token_id
1705
+ self.classifier = RobertaClassificationHead(config)
1706
+
1707
+ # Initialize weights and apply final processing
1708
+ self.post_init()
1709
+
1710
+
1711
+ @property
1712
+ def scaler(self):
1713
+ return self._scaler.exp()
1714
+
1715
+ @property
1716
+ def scaler_1(self):
1717
+ return self._scaler_1.exp()
1718
+
1719
+ @property
1720
+ def scaler_2(self):
1721
+ return self._scaler_2.exp()
1722
+
1723
+ @property
1724
+ def rel_weight(self):
1725
+ if self.config.weight_act == "sigmoid":
1726
+ return torch.sigmoid(self._rel_weight)
1727
+ elif self.config.weight_act == "softmax":
1728
+ return torch.softmax(self._rel_weight, dim=-1)
1729
+
1730
+ @property
1731
+ def rel_weight_1(self):
1732
+ if self.config.weight_act == "sigmoid":
1733
+ return torch.sigmoid(self._rel_weight_1)
1734
+ elif self.config.weight_act == "softmax":
1735
+ return torch.softmax(self._rel_weight_1, dim=-1)
1736
+
1737
+
1738
+ @property
1739
+ def rel_weight_2(self):
1740
+ if self.config.weight_act == "sigmoid":
1741
+ return torch.sigmoid(self._rel_weight_2)
1742
+ elif self.config.weight_act == "softmax":
1743
+ return torch.softmax(self._rel_weight_2, dim=-1)
1744
+
1745
+
1746
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1747
+ """Compute constituents from distance and height."""
1748
+
1749
+ if n_cntxt_layers>0:
1750
+ if n_cntxt_layers == 1:
1751
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1752
+ elif n_cntxt_layers == 2:
1753
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1754
+ else:
1755
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1756
+
1757
+ gamma = torch.sigmoid(-beta_logits)
1758
+ ones = torch.ones_like(gamma)
1759
+
1760
+ block_mask_left = cummin(
1761
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1762
+ )
1763
+ block_mask_left = block_mask_left - F.pad(
1764
+ block_mask_left[:, :, :-1], (1, 0), value=0
1765
+ )
1766
+ block_mask_left.tril_(0)
1767
+
1768
+ block_mask_right = cummin(
1769
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1770
+ )
1771
+ block_mask_right = block_mask_right - F.pad(
1772
+ block_mask_right[:, :, 1:], (0, 1), value=0
1773
+ )
1774
+ block_mask_right.triu_(0)
1775
+
1776
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1777
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1778
+ block_mask_right, reverse=True
1779
+ ).triu(1)
1780
+
1781
+ return block_p, block
1782
+
1783
+ def compute_head(self, height, n_cntxt_layers=0):
1784
+ """Estimate head for each constituent."""
1785
+
1786
+ _, length = height.size()
1787
+ if n_cntxt_layers>0:
1788
+ if n_cntxt_layers == 1:
1789
+ head_logits = height * self.scaler_1[1]
1790
+ elif n_cntxt_layers == 2:
1791
+ head_logits = height * self.scaler_2[1]
1792
+ else:
1793
+ head_logits = height * self.scaler[1]
1794
+ index = torch.arange(length, device=height.device)
1795
+
1796
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1797
+ index[None, None, :] <= index[None, :, None]
1798
+ )
1799
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1800
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1801
+
1802
+ head_p = torch.softmax(head_logits, dim=-1)
1803
+
1804
+ return head_p
1805
+
1806
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1807
+ """Parse input sentence.
1808
+
1809
+ Args:
1810
+ x: input tokens (required).
1811
+ pos: position for each token (optional).
1812
+ Returns:
1813
+ distance: syntactic distance
1814
+ height: syntactic height
1815
+ """
1816
+
1817
+ mask = x != self.pad
1818
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1819
+
1820
+ if embs is None:
1821
+ h = self.roberta.embeddings(x)
1822
+ else:
1823
+ h = embs
1824
+
1825
+ if n_cntxt_layers > 0:
1826
+ if n_cntxt_layers == 1:
1827
+ parser_layers = self.parser_layers_1
1828
+ height_ff = self.height_ff_1
1829
+ distance_ff = self.distance_ff_1
1830
+ elif n_cntxt_layers == 2:
1831
+ parser_layers = self.parser_layers_2
1832
+ height_ff = self.height_ff_2
1833
+ distance_ff = self.distance_ff_2
1834
+ for i in range(int(self.config.n_parser_layers/2)):
1835
+ h = h.masked_fill(~mask[:, :, None], 0)
1836
+ h = parser_layers[i](h)
1837
+
1838
+ height = height_ff(h).squeeze(-1)
1839
+ height.masked_fill_(~mask, -1e9)
1840
+
1841
+ distance = distance_ff(h).squeeze(-1)
1842
+ distance.masked_fill_(~mask_shifted, 1e9)
1843
+
1844
+ # Calbrating the distance and height to the same level
1845
+ length = distance.size(1)
1846
+ height_max = height[:, None, :].expand(-1, length, -1)
1847
+ height_max = torch.cummax(
1848
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1849
+ )[0].triu(0)
1850
+
1851
+ margin_left = torch.relu(
1852
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1853
+ )
1854
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1855
+ margin = torch.where(
1856
+ margin_left > margin_right, margin_right, margin_left
1857
+ ).triu(0)
1858
+
1859
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1860
+ margin.masked_fill_(~margin_mask, 0)
1861
+ margin = margin.max()
1862
+
1863
+ distance = distance - margin
1864
+ else:
1865
+ for i in range(self.config.n_parser_layers):
1866
+ h = h.masked_fill(~mask[:, :, None], 0)
1867
+ h = self.parser_layers[i](h)
1868
+
1869
+ height = self.height_ff(h).squeeze(-1)
1870
+ height.masked_fill_(~mask, -1e9)
1871
+
1872
+ distance = self.distance_ff(h).squeeze(-1)
1873
+ distance.masked_fill_(~mask_shifted, 1e9)
1874
+
1875
+ # Calbrating the distance and height to the same level
1876
+ length = distance.size(1)
1877
+ height_max = height[:, None, :].expand(-1, length, -1)
1878
+ height_max = torch.cummax(
1879
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1880
+ )[0].triu(0)
1881
+
1882
+ margin_left = torch.relu(
1883
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1884
+ )
1885
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1886
+ margin = torch.where(
1887
+ margin_left > margin_right, margin_right, margin_left
1888
+ ).triu(0)
1889
+
1890
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1891
+ margin.masked_fill_(~margin_mask, 0)
1892
+ margin = margin.max()
1893
+
1894
+ distance = distance - margin
1895
+
1896
+ return distance, height
1897
+
1898
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1899
+ """Compute head and cibling distribution for each token."""
1900
+
1901
+ bsz, length = x.size()
1902
+
1903
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1904
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1905
+
1906
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1907
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1908
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1909
+ head = head.masked_fill(eye, 0)
1910
+ child = head.transpose(1, 2)
1911
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1912
+
1913
+ rel_list = []
1914
+ if "head" in self.config.relations:
1915
+ rel_list.append(head)
1916
+ if "child" in self.config.relations:
1917
+ rel_list.append(child)
1918
+ if "cibling" in self.config.relations:
1919
+ rel_list.append(cibling)
1920
+
1921
+ rel = torch.stack(rel_list, dim=1)
1922
+
1923
+ if n_cntxt_layers > 0:
1924
+ if n_cntxt_layers == 1:
1925
+ rel_weight = self.rel_weight_1
1926
+ elif n_cntxt_layers == 2:
1927
+ rel_weight = self.rel_weight_2
1928
+ else:
1929
+ rel_weight = self.rel_weight
1930
+
1931
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1932
+
1933
+ if n_cntxt_layers == 1:
1934
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1935
+ else:
1936
+ num_layers = self.roberta.config.num_hidden_layers
1937
+
1938
+ att_mask = dep.reshape(
1939
+ num_layers,
1940
+ bsz,
1941
+ self.config.num_attention_heads,
1942
+ length,
1943
+ length,
1944
+ )
1945
+
1946
+ return att_mask, cibling, head, block
1947
+
1948
+ def forward(
1949
+ self,
1950
+ input_ids: Optional[torch.LongTensor] = None,
1951
+ attention_mask: Optional[torch.FloatTensor] = None,
1952
+ token_type_ids: Optional[torch.LongTensor] = None,
1953
+ position_ids: Optional[torch.LongTensor] = None,
1954
+ head_mask: Optional[torch.FloatTensor] = None,
1955
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1956
+ labels: Optional[torch.LongTensor] = None,
1957
+ output_attentions: Optional[bool] = None,
1958
+ output_hidden_states: Optional[bool] = None,
1959
+ return_dict: Optional[bool] = None,
1960
+ ) -> Union[Tuple, SequenceClassifierOutput]:
1961
+ r"""
1962
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1963
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1964
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1965
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1966
+ """
1967
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1968
+
1969
+ if self.config.n_cntxt_layers > 0:
1970
+ cntxt_outputs = self.cntxt_layers(
1971
+ input_ids,
1972
+ attention_mask=attention_mask,
1973
+ token_type_ids=token_type_ids,
1974
+ position_ids=position_ids,
1975
+ head_mask=head_mask,
1976
+ inputs_embeds=inputs_embeds,
1977
+ output_attentions=output_attentions,
1978
+ output_hidden_states=output_hidden_states,
1979
+ return_dict=return_dict)
1980
+
1981
+
1982
+ if self.config.n_cntxt_layers_2 > 0:
1983
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1984
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1985
+
1986
+ cntxt_outputs_2 = self.cntxt_layers_2(
1987
+ input_ids,
1988
+ attention_mask=attention_mask,
1989
+ token_type_ids=token_type_ids,
1990
+ position_ids=position_ids,
1991
+ head_mask=head_mask,
1992
+ inputs_embeds=inputs_embeds,
1993
+ output_attentions=output_attentions,
1994
+ output_hidden_states=output_hidden_states,
1995
+ return_dict=return_dict,
1996
+ parser_att_mask=att_mask_1)
1997
+
1998
+ sequence_output = cntxt_outputs_2[0]
1999
+
2000
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
2001
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
2002
+
2003
+ elif self.config.n_cntxt_layers > 0:
2004
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
2005
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2006
+ else:
2007
+ distance, height = self.parse(input_ids)
2008
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2009
+
2010
+ outputs = self.roberta(
2011
+ input_ids,
2012
+ attention_mask=attention_mask,
2013
+ token_type_ids=token_type_ids,
2014
+ position_ids=position_ids,
2015
+ head_mask=head_mask,
2016
+ inputs_embeds=inputs_embeds,
2017
+ output_attentions=output_attentions,
2018
+ output_hidden_states=output_hidden_states,
2019
+ return_dict=return_dict,
2020
+ parser_att_mask=att_mask,
2021
+ )
2022
+ sequence_output = outputs[0]
2023
+ logits = self.classifier(sequence_output)
2024
+
2025
+ loss = None
2026
+ if labels is not None:
2027
+ if self.config.problem_type is None:
2028
+ if self.num_labels == 1:
2029
+ self.config.problem_type = "regression"
2030
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
2031
+ self.config.problem_type = "single_label_classification"
2032
+ else:
2033
+ self.config.problem_type = "multi_label_classification"
2034
+
2035
+ if self.config.problem_type == "regression":
2036
+ loss_fct = MSELoss()
2037
+ if self.num_labels == 1:
2038
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
2039
+ else:
2040
+ loss = loss_fct(logits, labels)
2041
+ elif self.config.problem_type == "single_label_classification":
2042
+ loss_fct = CrossEntropyLoss()
2043
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
2044
+ elif self.config.problem_type == "multi_label_classification":
2045
+ loss_fct = BCEWithLogitsLoss()
2046
+ loss = loss_fct(logits, labels)
2047
+
2048
+ if not return_dict:
2049
+ output = (logits,) + outputs[2:]
2050
+ return ((loss,) + output) if loss is not None else output
2051
+
2052
+ return SequenceClassifierOutput(
2053
+ loss=loss,
2054
+ logits=logits,
2055
+ hidden_states=outputs.hidden_states,
2056
+ attentions=outputs.attentions,
2057
+ )
2058
+
2059
+
2060
+ class RobertaClassificationHead(nn.Module):
2061
+ """Head for sentence-level classification tasks."""
2062
+
2063
+ def __init__(self, config):
2064
+ super().__init__()
2065
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
2066
+ classifier_dropout = (
2067
+ config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
2068
+ )
2069
+ self.dropout = nn.Dropout(classifier_dropout)
2070
+ self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
2071
+
2072
+ def forward(self, features, **kwargs):
2073
+ x = features[:, 0, :] # take <s> token (equiv. to [CLS])
2074
+ x = self.dropout(x)
2075
+ x = self.dense(x)
2076
+ x = torch.tanh(x)
2077
+ x = self.dropout(x)
2078
+ x = self.out_proj(x)
2079
+ return x
2080
+
2081
+
2082
+ def create_position_ids_from_input_ids(
2083
+ input_ids, padding_idx, past_key_values_length=0
2084
+ ):
2085
+ """
2086
+ Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
2087
+ are ignored. This is modified from fairseq's `utils.make_positions`.
2088
+
2089
+ Args:
2090
+ x: torch.Tensor x:
2091
+
2092
+ Returns: torch.Tensor
2093
+ """
2094
+ # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
2095
+ mask = input_ids.ne(padding_idx).int()
2096
+ incremental_indices = (
2097
+ torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length
2098
+ ) * mask
2099
+ return incremental_indices.long() + padding_idx
2100
+
2101
+
2102
+ def cumprod(x, reverse=False, exclusive=False):
2103
+ """cumulative product."""
2104
+ if reverse:
2105
+ x = x.flip([-1])
2106
+
2107
+ if exclusive:
2108
+ x = F.pad(x[:, :, :-1], (1, 0), value=1)
2109
+
2110
+ cx = x.cumprod(-1)
2111
+
2112
+ if reverse:
2113
+ cx = cx.flip([-1])
2114
+ return cx
2115
+
2116
+
2117
+ def cumsum(x, reverse=False, exclusive=False):
2118
+ """cumulative sum."""
2119
+ bsz, _, length = x.size()
2120
+ device = x.device
2121
+ if reverse:
2122
+ if exclusive:
2123
+ w = torch.ones([bsz, length, length], device=device).tril(-1)
2124
+ else:
2125
+ w = torch.ones([bsz, length, length], device=device).tril(0)
2126
+ cx = torch.bmm(x, w)
2127
+ else:
2128
+ if exclusive:
2129
+ w = torch.ones([bsz, length, length], device=device).triu(1)
2130
+ else:
2131
+ w = torch.ones([bsz, length, length], device=device).triu(0)
2132
+ cx = torch.bmm(x, w)
2133
+ return cx
2134
+
2135
+
2136
+ def cummin(x, reverse=False, exclusive=False, max_value=1e9):
2137
+ """cumulative min."""
2138
+ if reverse:
2139
+ if exclusive:
2140
+ x = F.pad(x[:, :, 1:], (0, 1), value=max_value)
2141
+ x = x.flip([-1]).cummin(-1)[0].flip([-1])
2142
+ else:
2143
+ if exclusive:
2144
+ x = F.pad(x[:, :, :-1], (1, 0), value=max_value)
2145
+ x = x.cummin(-1)[0]
2146
+ return x
finetune/control_raising_control/predict_results.txt ADDED
The diff for this file is too large to render. See raw diff
 
finetune/control_raising_control/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f315303adbb6e1af6f1c2fc7b1e2fb76a659dd9640b64fde81698aeef43a1f86
3
+ size 534594825
finetune/control_raising_control/special_tokens_map.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "mask_token": {
6
+ "content": "<mask>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "pad_token": "<pad>",
13
+ "sep_token": "</s>",
14
+ "unk_token": "<unk>"
15
+ }
finetune/control_raising_control/tokenizer_config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": {
4
+ "__type": "AddedToken",
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ },
11
+ "cls_token": {
12
+ "__type": "AddedToken",
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "errors": "replace",
28
+ "mask_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<mask>",
31
+ "lstrip": true,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false
35
+ },
36
+ "model_max_length": 512,
37
+ "name_or_path": "final_models/glue_models/structroberta_s2_final/",
38
+ "pad_token": {
39
+ "__type": "AddedToken",
40
+ "content": "<pad>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": false
45
+ },
46
+ "sep_token": {
47
+ "__type": "AddedToken",
48
+ "content": "</s>",
49
+ "lstrip": false,
50
+ "normalized": true,
51
+ "rstrip": false,
52
+ "single_word": false
53
+ },
54
+ "special_tokens_map_file": null,
55
+ "tokenizer_class": "RobertaTokenizer",
56
+ "trim_offsets": true,
57
+ "unk_token": {
58
+ "__type": "AddedToken",
59
+ "content": "<unk>",
60
+ "lstrip": false,
61
+ "normalized": true,
62
+ "rstrip": false,
63
+ "single_word": false
64
+ }
65
+ }
finetune/control_raising_control/train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "train_loss": 0.02885379483330656,
4
+ "train_runtime": 310.6244,
5
+ "train_samples": 6570,
6
+ "train_samples_per_second": 211.509,
7
+ "train_steps_per_second": 1.771
8
+ }
finetune/control_raising_control/trainer_state.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.889028038683643,
3
+ "best_model_checkpoint": "final_models/glue_models/structroberta_s2_final//finetune/control_raising_control/checkpoint-400",
4
+ "epoch": 10.0,
5
+ "global_step": 550,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 7.27,
12
+ "eval_accuracy": 0.880810022354126,
13
+ "eval_f1": 0.889028038683643,
14
+ "eval_loss": 0.8526787161827087,
15
+ "eval_mcc": 0.7731937974254972,
16
+ "eval_runtime": 26.0182,
17
+ "eval_samples_per_second": 514.332,
18
+ "eval_steps_per_second": 64.301,
19
+ "step": 400
20
+ },
21
+ {
22
+ "epoch": 9.09,
23
+ "learning_rate": 4.5454545454545455e-06,
24
+ "loss": 0.0317,
25
+ "step": 500
26
+ },
27
+ {
28
+ "epoch": 10.0,
29
+ "step": 550,
30
+ "total_flos": 5482602020505600.0,
31
+ "train_loss": 0.02885379483330656,
32
+ "train_runtime": 310.6244,
33
+ "train_samples_per_second": 211.509,
34
+ "train_steps_per_second": 1.771
35
+ }
36
+ ],
37
+ "max_steps": 550,
38
+ "num_train_epochs": 10,
39
+ "total_flos": 5482602020505600.0,
40
+ "trial_name": null,
41
+ "trial_params": null
42
+ }
finetune/control_raising_control/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb0f4ddd57425df2852cd6f09df0e748a99db4d4c191d3ae51520e8bebf8a0fd
3
+ size 3567
finetune/control_raising_control/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
finetune/control_raising_lexical_content_the/all_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.6656312346458435,
4
+ "eval_f1": 0.494088272012925,
5
+ "eval_loss": 3.040160655975342,
6
+ "eval_mcc": 0.43895218658919233,
7
+ "eval_runtime": 40.0677,
8
+ "eval_samples": 20603,
9
+ "eval_samples_per_second": 514.205,
10
+ "eval_steps_per_second": 64.291,
11
+ "train_loss": 0.011623076279239173,
12
+ "train_runtime": 335.2215,
13
+ "train_samples": 6816,
14
+ "train_samples_per_second": 203.328,
15
+ "train_steps_per_second": 1.7
16
+ }
finetune/control_raising_lexical_content_the/config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "final_models/glue_models/structroberta_s2_final/",
3
+ "architectures": [
4
+ "StructRobertaForSequenceClassification"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "auto_map": {
8
+ "AutoConfig": "modeling_structroberta.StructRobertaConfig",
9
+ "AutoModelForMaskedLM": "modeling_structroberta.StructRoberta",
10
+ "AutoModelForSequenceClassification": "modeling_structroberta.StructRobertaForSequenceClassification"
11
+ },
12
+ "bos_token_id": 0,
13
+ "classifier_dropout": null,
14
+ "conv_size": 9,
15
+ "eos_token_id": 2,
16
+ "hidden_act": "gelu",
17
+ "hidden_dropout_prob": 0.1,
18
+ "hidden_size": 768,
19
+ "id2label": {
20
+ "0": 0,
21
+ "1": 1
22
+ },
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 3072,
25
+ "label2id": {
26
+ "0": 0,
27
+ "1": 1
28
+ },
29
+ "layer_norm_eps": 1e-05,
30
+ "max_position_embeddings": 514,
31
+ "model_type": "roberta",
32
+ "n_cntxt_layers": 4,
33
+ "n_cntxt_layers_2": 0,
34
+ "n_parser_layers": 4,
35
+ "num_attention_heads": 12,
36
+ "num_hidden_layers": 8,
37
+ "pad_token_id": 1,
38
+ "position_embedding_type": "absolute",
39
+ "problem_type": "single_label_classification",
40
+ "relations": [
41
+ "head",
42
+ "child"
43
+ ],
44
+ "torch_dtype": "float32",
45
+ "transformers_version": "4.26.1",
46
+ "type_vocab_size": 1,
47
+ "use_cache": true,
48
+ "vocab_size": 32000,
49
+ "weight_act": "softmax"
50
+ }
finetune/control_raising_lexical_content_the/eval_results.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.6656312346458435,
4
+ "eval_f1": 0.494088272012925,
5
+ "eval_loss": 3.040160655975342,
6
+ "eval_mcc": 0.43895218658919233,
7
+ "eval_runtime": 40.0677,
8
+ "eval_samples": 20603,
9
+ "eval_samples_per_second": 514.205,
10
+ "eval_steps_per_second": 64.291
11
+ }
finetune/control_raising_lexical_content_the/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
finetune/control_raising_lexical_content_the/modeling_structroberta.py ADDED
@@ -0,0 +1,2146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import math
3
+ from typing import List, Optional, Tuple, Union
4
+ import torch
5
+ import torch.utils.checkpoint
6
+ from packaging import version
7
+ from torch import nn
8
+ import torch.nn.functional as F
9
+ from torch.nn import CrossEntropyLoss, MSELoss, BCEWithLogitsLoss
10
+ from transformers.activations import ACT2FN, gelu
11
+ from transformers.modeling_outputs import (
12
+ BaseModelOutputWithPastAndCrossAttentions,
13
+ BaseModelOutputWithPoolingAndCrossAttentions,
14
+ MaskedLMOutput,
15
+ SequenceClassifierOutput
16
+ )
17
+ from transformers.modeling_utils import (
18
+ PreTrainedModel,
19
+ apply_chunking_to_forward,
20
+ find_pruneable_heads_and_indices,
21
+ prune_linear_layer,
22
+ )
23
+ from transformers.utils import logging
24
+ from transformers import RobertaConfig
25
+
26
+
27
+ logger = logging.get_logger(__name__)
28
+ ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [
29
+ "roberta-base",
30
+ "roberta-large",
31
+ "roberta-large-mnli",
32
+ "distilroberta-base",
33
+ "roberta-base-openai-detector",
34
+ "roberta-large-openai-detector",
35
+ # See all RoBERTa models at https://huggingface.co/models?filter=roberta
36
+ ]
37
+
38
+
39
+ class StructRobertaConfig(RobertaConfig):
40
+ model_type = "roberta"
41
+
42
+ def __init__(
43
+ self,
44
+ n_parser_layers=4,
45
+ conv_size=9,
46
+ relations=("head", "child"),
47
+ weight_act="softmax",
48
+ n_cntxt_layers=3,
49
+ n_cntxt_layers_2=0,
50
+ **kwargs,):
51
+
52
+ super().__init__(**kwargs)
53
+ self.n_cntxt_layers = n_cntxt_layers
54
+ self.n_parser_layers = n_parser_layers
55
+ self.n_cntxt_layers_2 = n_cntxt_layers_2
56
+ self.conv_size = conv_size
57
+ self.relations = relations
58
+ self.weight_act = weight_act
59
+
60
+ class Conv1d(nn.Module):
61
+ """1D convolution layer."""
62
+
63
+ def __init__(self, hidden_size, kernel_size, dilation=1):
64
+ """Initialization.
65
+
66
+ Args:
67
+ hidden_size: dimension of input embeddings
68
+ kernel_size: convolution kernel size
69
+ dilation: the spacing between the kernel points
70
+ """
71
+ super(Conv1d, self).__init__()
72
+
73
+ if kernel_size % 2 == 0:
74
+ padding = (kernel_size // 2) * dilation
75
+ self.shift = True
76
+ else:
77
+ padding = ((kernel_size - 1) // 2) * dilation
78
+ self.shift = False
79
+ self.conv = nn.Conv1d(
80
+ hidden_size, hidden_size, kernel_size, padding=padding, dilation=dilation
81
+ )
82
+
83
+ def forward(self, x):
84
+ """Compute convolution.
85
+
86
+ Args:
87
+ x: input embeddings
88
+ Returns:
89
+ conv_output: convolution results
90
+ """
91
+
92
+ if self.shift:
93
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)[:, 1:]
94
+ else:
95
+ return self.conv(x.transpose(1, 2)).transpose(1, 2)
96
+
97
+
98
+ class RobertaEmbeddings(nn.Module):
99
+ """
100
+ Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
101
+ """
102
+
103
+ # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__
104
+ def __init__(self, config):
105
+ super().__init__()
106
+ self.word_embeddings = nn.Embedding(
107
+ config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id
108
+ )
109
+ self.position_embeddings = nn.Embedding(
110
+ config.max_position_embeddings, config.hidden_size
111
+ )
112
+ self.token_type_embeddings = nn.Embedding(
113
+ config.type_vocab_size, config.hidden_size
114
+ )
115
+
116
+ # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
117
+ # any TensorFlow checkpoint file
118
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
119
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
120
+ # position_ids (1, len position emb) is contiguous in memory and exported when serialized
121
+ self.position_embedding_type = getattr(
122
+ config, "position_embedding_type", "absolute"
123
+ )
124
+ self.register_buffer(
125
+ "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))
126
+ )
127
+ if version.parse(torch.__version__) > version.parse("1.6.0"):
128
+ self.register_buffer(
129
+ "token_type_ids",
130
+ torch.zeros(self.position_ids.size(), dtype=torch.long),
131
+ persistent=False,
132
+ )
133
+
134
+ # End copy
135
+ self.padding_idx = config.pad_token_id
136
+ self.position_embeddings = nn.Embedding(
137
+ config.max_position_embeddings,
138
+ config.hidden_size,
139
+ padding_idx=self.padding_idx,
140
+ )
141
+
142
+ def forward(
143
+ self,
144
+ input_ids=None,
145
+ token_type_ids=None,
146
+ position_ids=None,
147
+ inputs_embeds=None,
148
+ past_key_values_length=0,
149
+ ):
150
+ if position_ids is None:
151
+ if input_ids is not None:
152
+ # Create the position ids from the input token ids. Any padded tokens remain padded.
153
+ position_ids = create_position_ids_from_input_ids(
154
+ input_ids, self.padding_idx, past_key_values_length
155
+ )
156
+ else:
157
+ position_ids = self.create_position_ids_from_inputs_embeds(
158
+ inputs_embeds
159
+ )
160
+
161
+ if input_ids is not None:
162
+ input_shape = input_ids.size()
163
+ else:
164
+ input_shape = inputs_embeds.size()[:-1]
165
+
166
+ seq_length = input_shape[1]
167
+
168
+ # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs
169
+ # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
170
+ # issue #5664
171
+ if token_type_ids is None:
172
+ if hasattr(self, "token_type_ids"):
173
+ buffered_token_type_ids = self.token_type_ids[:, :seq_length]
174
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
175
+ input_shape[0], seq_length
176
+ )
177
+ token_type_ids = buffered_token_type_ids_expanded
178
+ else:
179
+ token_type_ids = torch.zeros(
180
+ input_shape, dtype=torch.long, device=self.position_ids.device
181
+ )
182
+
183
+ if inputs_embeds is None:
184
+ inputs_embeds = self.word_embeddings(input_ids)
185
+ token_type_embeddings = self.token_type_embeddings(token_type_ids)
186
+
187
+ embeddings = inputs_embeds + token_type_embeddings
188
+ if self.position_embedding_type == "absolute":
189
+ position_embeddings = self.position_embeddings(position_ids)
190
+ embeddings += position_embeddings
191
+ embeddings = self.LayerNorm(embeddings)
192
+ embeddings = self.dropout(embeddings)
193
+ return embeddings
194
+
195
+ def create_position_ids_from_inputs_embeds(self, inputs_embeds):
196
+ """
197
+ We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.
198
+
199
+ Args:
200
+ inputs_embeds: torch.Tensor
201
+
202
+ Returns: torch.Tensor
203
+ """
204
+ input_shape = inputs_embeds.size()[:-1]
205
+ sequence_length = input_shape[1]
206
+
207
+ position_ids = torch.arange(
208
+ self.padding_idx + 1,
209
+ sequence_length + self.padding_idx + 1,
210
+ dtype=torch.long,
211
+ device=inputs_embeds.device,
212
+ )
213
+ return position_ids.unsqueeze(0).expand(input_shape)
214
+
215
+
216
+ # Copied from transformers.models.bert.modeling_bert.BertSelfAttention with Bert->Roberta
217
+ class RobertaSelfAttention(nn.Module):
218
+ def __init__(self, config, position_embedding_type=None):
219
+ super().__init__()
220
+ if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
221
+ config, "embedding_size"
222
+ ):
223
+ raise ValueError(
224
+ f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
225
+ f"heads ({config.num_attention_heads})"
226
+ )
227
+
228
+ self.num_attention_heads = config.num_attention_heads
229
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
230
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
231
+
232
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
233
+ self.key = nn.Linear(config.hidden_size, self.all_head_size)
234
+ self.value = nn.Linear(config.hidden_size, self.all_head_size)
235
+
236
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
237
+ self.position_embedding_type = position_embedding_type or getattr(
238
+ config, "position_embedding_type", "absolute"
239
+ )
240
+ if (
241
+ self.position_embedding_type == "relative_key"
242
+ or self.position_embedding_type == "relative_key_query"
243
+ ):
244
+ self.max_position_embeddings = config.max_position_embeddings
245
+ self.distance_embedding = nn.Embedding(
246
+ 2 * config.max_position_embeddings - 1, self.attention_head_size
247
+ )
248
+
249
+ self.is_decoder = config.is_decoder
250
+
251
+ def transpose_for_scores(self, x):
252
+ new_x_shape = x.size()[:-1] + (
253
+ self.num_attention_heads,
254
+ self.attention_head_size,
255
+ )
256
+ x = x.view(new_x_shape)
257
+ return x.permute(0, 2, 1, 3)
258
+
259
+ def forward(
260
+ self,
261
+ hidden_states: torch.Tensor,
262
+ attention_mask: Optional[torch.FloatTensor] = None,
263
+ head_mask: Optional[torch.FloatTensor] = None,
264
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
265
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
266
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
267
+ output_attentions: Optional[bool] = False,
268
+ parser_att_mask=None,
269
+ ) -> Tuple[torch.Tensor]:
270
+ mixed_query_layer = self.query(hidden_states)
271
+
272
+ # If this is instantiated as a cross-attention module, the keys
273
+ # and values come from an encoder; the attention mask needs to be
274
+ # such that the encoder's padding tokens are not attended to.
275
+ is_cross_attention = encoder_hidden_states is not None
276
+
277
+ if is_cross_attention and past_key_value is not None:
278
+ # reuse k,v, cross_attentions
279
+ key_layer = past_key_value[0]
280
+ value_layer = past_key_value[1]
281
+ attention_mask = encoder_attention_mask
282
+ elif is_cross_attention:
283
+ key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
284
+ value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
285
+ attention_mask = encoder_attention_mask
286
+ elif past_key_value is not None:
287
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
288
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
289
+ key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
290
+ value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
291
+ else:
292
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
293
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
294
+
295
+ query_layer = self.transpose_for_scores(mixed_query_layer)
296
+
297
+ if self.is_decoder:
298
+ # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
299
+ # Further calls to cross_attention layer can then reuse all cross-attention
300
+ # key/value_states (first "if" case)
301
+ # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
302
+ # all previous decoder key/value_states. Further calls to uni-directional self-attention
303
+ # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
304
+ # if encoder bi-directional self-attention `past_key_value` is always `None`
305
+ past_key_value = (key_layer, value_layer)
306
+
307
+ # Take the dot product between "query" and "key" to get the raw attention scores.
308
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
309
+
310
+ if (
311
+ self.position_embedding_type == "relative_key"
312
+ or self.position_embedding_type == "relative_key_query"
313
+ ):
314
+ seq_length = hidden_states.size()[1]
315
+ position_ids_l = torch.arange(
316
+ seq_length, dtype=torch.long, device=hidden_states.device
317
+ ).view(-1, 1)
318
+ position_ids_r = torch.arange(
319
+ seq_length, dtype=torch.long, device=hidden_states.device
320
+ ).view(1, -1)
321
+ distance = position_ids_l - position_ids_r
322
+ positional_embedding = self.distance_embedding(
323
+ distance + self.max_position_embeddings - 1
324
+ )
325
+ positional_embedding = positional_embedding.to(
326
+ dtype=query_layer.dtype
327
+ ) # fp16 compatibility
328
+
329
+ if self.position_embedding_type == "relative_key":
330
+ relative_position_scores = torch.einsum(
331
+ "bhld,lrd->bhlr", query_layer, positional_embedding
332
+ )
333
+ attention_scores = attention_scores + relative_position_scores
334
+ elif self.position_embedding_type == "relative_key_query":
335
+ relative_position_scores_query = torch.einsum(
336
+ "bhld,lrd->bhlr", query_layer, positional_embedding
337
+ )
338
+ relative_position_scores_key = torch.einsum(
339
+ "bhrd,lrd->bhlr", key_layer, positional_embedding
340
+ )
341
+ attention_scores = (
342
+ attention_scores
343
+ + relative_position_scores_query
344
+ + relative_position_scores_key
345
+ )
346
+
347
+ attention_scores = attention_scores / math.sqrt(self.attention_head_size)
348
+ if attention_mask is not None:
349
+ # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
350
+ attention_scores = attention_scores + attention_mask
351
+
352
+ if parser_att_mask is None:
353
+ # Normalize the attention scores to probabilities.
354
+ attention_probs = nn.functional.softmax(attention_scores, dim=-1)
355
+ else:
356
+ attention_probs = torch.sigmoid(attention_scores) * parser_att_mask
357
+
358
+ # This is actually dropping out entire tokens to attend to, which might
359
+ # seem a bit unusual, but is taken from the original Transformer paper.
360
+ attention_probs = self.dropout(attention_probs)
361
+
362
+ # Mask heads if we want to
363
+ if head_mask is not None:
364
+ attention_probs = attention_probs * head_mask
365
+
366
+ context_layer = torch.matmul(attention_probs, value_layer)
367
+
368
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
369
+ new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
370
+ context_layer = context_layer.view(new_context_layer_shape)
371
+
372
+ outputs = (
373
+ (context_layer, attention_probs) if output_attentions else (context_layer,)
374
+ )
375
+
376
+ if self.is_decoder:
377
+ outputs = outputs + (past_key_value,)
378
+ return outputs
379
+
380
+
381
+ # Copied from transformers.models.bert.modeling_bert.BertSelfOutput
382
+ class RobertaSelfOutput(nn.Module):
383
+ def __init__(self, config):
384
+ super().__init__()
385
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
386
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
387
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
388
+
389
+ def forward(
390
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
391
+ ) -> torch.Tensor:
392
+ hidden_states = self.dense(hidden_states)
393
+ hidden_states = self.dropout(hidden_states)
394
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
395
+ return hidden_states
396
+
397
+
398
+ # Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Roberta
399
+ class RobertaAttention(nn.Module):
400
+ def __init__(self, config, position_embedding_type=None):
401
+ super().__init__()
402
+ self.self = RobertaSelfAttention(
403
+ config, position_embedding_type=position_embedding_type
404
+ )
405
+ self.output = RobertaSelfOutput(config)
406
+ self.pruned_heads = set()
407
+
408
+ def prune_heads(self, heads):
409
+ if len(heads) == 0:
410
+ return
411
+ heads, index = find_pruneable_heads_and_indices(
412
+ heads,
413
+ self.self.num_attention_heads,
414
+ self.self.attention_head_size,
415
+ self.pruned_heads,
416
+ )
417
+
418
+ # Prune linear layers
419
+ self.self.query = prune_linear_layer(self.self.query, index)
420
+ self.self.key = prune_linear_layer(self.self.key, index)
421
+ self.self.value = prune_linear_layer(self.self.value, index)
422
+ self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
423
+
424
+ # Update hyper params and store pruned heads
425
+ self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
426
+ self.self.all_head_size = (
427
+ self.self.attention_head_size * self.self.num_attention_heads
428
+ )
429
+ self.pruned_heads = self.pruned_heads.union(heads)
430
+
431
+ def forward(
432
+ self,
433
+ hidden_states: torch.Tensor,
434
+ attention_mask: Optional[torch.FloatTensor] = None,
435
+ head_mask: Optional[torch.FloatTensor] = None,
436
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
437
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
438
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
439
+ output_attentions: Optional[bool] = False,
440
+ parser_att_mask=None,
441
+ ) -> Tuple[torch.Tensor]:
442
+ self_outputs = self.self(
443
+ hidden_states,
444
+ attention_mask,
445
+ head_mask,
446
+ encoder_hidden_states,
447
+ encoder_attention_mask,
448
+ past_key_value,
449
+ output_attentions,
450
+ parser_att_mask=parser_att_mask,
451
+ )
452
+ attention_output = self.output(self_outputs[0], hidden_states)
453
+ outputs = (attention_output,) + self_outputs[
454
+ 1:
455
+ ] # add attentions if we output them
456
+ return outputs
457
+
458
+
459
+ # Copied from transformers.models.bert.modeling_bert.BertIntermediate
460
+ class RobertaIntermediate(nn.Module):
461
+ def __init__(self, config):
462
+ super().__init__()
463
+ self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
464
+ if isinstance(config.hidden_act, str):
465
+ self.intermediate_act_fn = ACT2FN[config.hidden_act]
466
+ else:
467
+ self.intermediate_act_fn = config.hidden_act
468
+
469
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
470
+ hidden_states = self.dense(hidden_states)
471
+ hidden_states = self.intermediate_act_fn(hidden_states)
472
+ return hidden_states
473
+
474
+
475
+ # Copied from transformers.models.bert.modeling_bert.BertOutput
476
+ class RobertaOutput(nn.Module):
477
+ def __init__(self, config):
478
+ super().__init__()
479
+ self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
480
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
481
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
482
+
483
+ def forward(
484
+ self, hidden_states: torch.Tensor, input_tensor: torch.Tensor
485
+ ) -> torch.Tensor:
486
+ hidden_states = self.dense(hidden_states)
487
+ hidden_states = self.dropout(hidden_states)
488
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
489
+ return hidden_states
490
+
491
+
492
+ # Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Roberta
493
+ class RobertaLayer(nn.Module):
494
+ def __init__(self, config):
495
+ super().__init__()
496
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
497
+ self.seq_len_dim = 1
498
+ self.attention = RobertaAttention(config)
499
+ self.is_decoder = config.is_decoder
500
+ self.add_cross_attention = config.add_cross_attention
501
+ if self.add_cross_attention:
502
+ if not self.is_decoder:
503
+ raise ValueError(
504
+ f"{self} should be used as a decoder model if cross attention is added"
505
+ )
506
+ self.crossattention = RobertaAttention(
507
+ config, position_embedding_type="absolute"
508
+ )
509
+ self.intermediate = RobertaIntermediate(config)
510
+ self.output = RobertaOutput(config)
511
+
512
+ def forward(
513
+ self,
514
+ hidden_states: torch.Tensor,
515
+ attention_mask: Optional[torch.FloatTensor] = None,
516
+ head_mask: Optional[torch.FloatTensor] = None,
517
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
518
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
519
+ past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
520
+ output_attentions: Optional[bool] = False,
521
+ parser_att_mask=None,
522
+ ) -> Tuple[torch.Tensor]:
523
+ # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
524
+ self_attn_past_key_value = (
525
+ past_key_value[:2] if past_key_value is not None else None
526
+ )
527
+ self_attention_outputs = self.attention(
528
+ hidden_states,
529
+ attention_mask,
530
+ head_mask,
531
+ output_attentions=output_attentions,
532
+ past_key_value=self_attn_past_key_value,
533
+ parser_att_mask=parser_att_mask,
534
+ )
535
+ attention_output = self_attention_outputs[0]
536
+
537
+ # if decoder, the last output is tuple of self-attn cache
538
+ if self.is_decoder:
539
+ outputs = self_attention_outputs[1:-1]
540
+ present_key_value = self_attention_outputs[-1]
541
+ else:
542
+ outputs = self_attention_outputs[
543
+ 1:
544
+ ] # add self attentions if we output attention weights
545
+
546
+ cross_attn_present_key_value = None
547
+ if self.is_decoder and encoder_hidden_states is not None:
548
+ if not hasattr(self, "crossattention"):
549
+ raise ValueError(
550
+ f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`"
551
+ )
552
+
553
+ # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
554
+ cross_attn_past_key_value = (
555
+ past_key_value[-2:] if past_key_value is not None else None
556
+ )
557
+ cross_attention_outputs = self.crossattention(
558
+ attention_output,
559
+ attention_mask,
560
+ head_mask,
561
+ encoder_hidden_states,
562
+ encoder_attention_mask,
563
+ cross_attn_past_key_value,
564
+ output_attentions,
565
+ )
566
+ attention_output = cross_attention_outputs[0]
567
+ outputs = (
568
+ outputs + cross_attention_outputs[1:-1]
569
+ ) # add cross attentions if we output attention weights
570
+
571
+ # add cross-attn cache to positions 3,4 of present_key_value tuple
572
+ cross_attn_present_key_value = cross_attention_outputs[-1]
573
+ present_key_value = present_key_value + cross_attn_present_key_value
574
+
575
+ layer_output = apply_chunking_to_forward(
576
+ self.feed_forward_chunk,
577
+ self.chunk_size_feed_forward,
578
+ self.seq_len_dim,
579
+ attention_output,
580
+ )
581
+ outputs = (layer_output,) + outputs
582
+
583
+ # if decoder, return the attn key/values as the last output
584
+ if self.is_decoder:
585
+ outputs = outputs + (present_key_value,)
586
+
587
+ return outputs
588
+
589
+ def feed_forward_chunk(self, attention_output):
590
+ intermediate_output = self.intermediate(attention_output)
591
+ layer_output = self.output(intermediate_output, attention_output)
592
+ return layer_output
593
+
594
+
595
+ # Copied from transformers.models.bert.modeling_bert.BertEncoder with Bert->Roberta
596
+ class RobertaEncoder(nn.Module):
597
+ def __init__(self, config):
598
+ super().__init__()
599
+ self.config = config
600
+ self.layer = nn.ModuleList(
601
+ [RobertaLayer(config) for _ in range(config.num_hidden_layers)]
602
+ )
603
+ self.gradient_checkpointing = False
604
+
605
+ def forward(
606
+ self,
607
+ hidden_states: torch.Tensor,
608
+ attention_mask: Optional[torch.FloatTensor] = None,
609
+ head_mask: Optional[torch.FloatTensor] = None,
610
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
611
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
612
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
613
+ use_cache: Optional[bool] = None,
614
+ output_attentions: Optional[bool] = False,
615
+ output_hidden_states: Optional[bool] = False,
616
+ return_dict: Optional[bool] = True,
617
+ parser_att_mask=None,
618
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]:
619
+ all_hidden_states = () if output_hidden_states else None
620
+ all_self_attentions = () if output_attentions else None
621
+ all_cross_attentions = (
622
+ () if output_attentions and self.config.add_cross_attention else None
623
+ )
624
+
625
+ next_decoder_cache = () if use_cache else None
626
+ for i, layer_module in enumerate(self.layer):
627
+ if output_hidden_states:
628
+ all_hidden_states = all_hidden_states + (hidden_states,)
629
+
630
+ layer_head_mask = head_mask[i] if head_mask is not None else None
631
+ past_key_value = past_key_values[i] if past_key_values is not None else None
632
+
633
+ if self.gradient_checkpointing and self.training:
634
+
635
+ if use_cache:
636
+ logger.warning(
637
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
638
+ )
639
+ use_cache = False
640
+
641
+ def create_custom_forward(module):
642
+ def custom_forward(*inputs):
643
+ return module(*inputs, past_key_value, output_attentions)
644
+
645
+ return custom_forward
646
+
647
+ layer_outputs = torch.utils.checkpoint.checkpoint(
648
+ create_custom_forward(layer_module),
649
+ hidden_states,
650
+ attention_mask,
651
+ layer_head_mask,
652
+ encoder_hidden_states,
653
+ encoder_attention_mask,
654
+ )
655
+ else:
656
+ if parser_att_mask is not None:
657
+ layer_outputs = layer_module(
658
+ hidden_states,
659
+ attention_mask,
660
+ layer_head_mask,
661
+ encoder_hidden_states,
662
+ encoder_attention_mask,
663
+ past_key_value,
664
+ output_attentions,
665
+ parser_att_mask=parser_att_mask[i])
666
+ else:
667
+ layer_outputs = layer_module(
668
+ hidden_states,
669
+ attention_mask,
670
+ layer_head_mask,
671
+ encoder_hidden_states,
672
+ encoder_attention_mask,
673
+ past_key_value,
674
+ output_attentions,
675
+ parser_att_mask=None)
676
+
677
+
678
+ hidden_states = layer_outputs[0]
679
+ if use_cache:
680
+ next_decoder_cache += (layer_outputs[-1],)
681
+ if output_attentions:
682
+ all_self_attentions = all_self_attentions + (layer_outputs[1],)
683
+ if self.config.add_cross_attention:
684
+ all_cross_attentions = all_cross_attentions + (layer_outputs[2],)
685
+
686
+ if output_hidden_states:
687
+ all_hidden_states = all_hidden_states + (hidden_states,)
688
+
689
+ if not return_dict:
690
+ return tuple(
691
+ v
692
+ for v in [
693
+ hidden_states,
694
+ next_decoder_cache,
695
+ all_hidden_states,
696
+ all_self_attentions,
697
+ all_cross_attentions,
698
+ ]
699
+ if v is not None
700
+ )
701
+ return BaseModelOutputWithPastAndCrossAttentions(
702
+ last_hidden_state=hidden_states,
703
+ past_key_values=next_decoder_cache,
704
+ hidden_states=all_hidden_states,
705
+ attentions=all_self_attentions,
706
+ cross_attentions=all_cross_attentions,
707
+ )
708
+
709
+
710
+ # Copied from transformers.models.bert.modeling_bert.BertPooler
711
+ class RobertaPooler(nn.Module):
712
+ def __init__(self, config):
713
+ super().__init__()
714
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
715
+ self.activation = nn.Tanh()
716
+
717
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
718
+ # We "pool" the model by simply taking the hidden state corresponding
719
+ # to the first token.
720
+ first_token_tensor = hidden_states[:, 0]
721
+ pooled_output = self.dense(first_token_tensor)
722
+ pooled_output = self.activation(pooled_output)
723
+ return pooled_output
724
+
725
+
726
+ class RobertaPreTrainedModel(PreTrainedModel):
727
+ """
728
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
729
+ models.
730
+ """
731
+
732
+ config_class = RobertaConfig
733
+ base_model_prefix = "roberta"
734
+ supports_gradient_checkpointing = True
735
+
736
+ # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights
737
+ def _init_weights(self, module):
738
+ """Initialize the weights"""
739
+ if isinstance(module, nn.Linear):
740
+ # Slightly different from the TF version which uses truncated_normal for initialization
741
+ # cf https://github.com/pytorch/pytorch/pull/5617
742
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
743
+ if module.bias is not None:
744
+ module.bias.data.zero_()
745
+ elif isinstance(module, nn.Embedding):
746
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
747
+ if module.padding_idx is not None:
748
+ module.weight.data[module.padding_idx].zero_()
749
+ elif isinstance(module, nn.LayerNorm):
750
+ if module.bias is not None:
751
+ module.bias.data.zero_()
752
+ module.weight.data.fill_(1.0)
753
+
754
+ def _set_gradient_checkpointing(self, module, value=False):
755
+ if isinstance(module, RobertaEncoder):
756
+ module.gradient_checkpointing = value
757
+
758
+ def update_keys_to_ignore(self, config, del_keys_to_ignore):
759
+ """Remove some keys from ignore list"""
760
+ if not config.tie_word_embeddings:
761
+ # must make a new list, or the class variable gets modified!
762
+ self._keys_to_ignore_on_save = [
763
+ k for k in self._keys_to_ignore_on_save if k not in del_keys_to_ignore
764
+ ]
765
+ self._keys_to_ignore_on_load_missing = [
766
+ k
767
+ for k in self._keys_to_ignore_on_load_missing
768
+ if k not in del_keys_to_ignore
769
+ ]
770
+
771
+
772
+ ROBERTA_START_DOCSTRING = r"""
773
+
774
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
775
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
776
+ etc.)
777
+
778
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
779
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
780
+ and behavior.
781
+
782
+ Parameters:
783
+ config ([`RobertaConfig`]): Model configuration class with all the parameters of the
784
+ model. Initializing with a config file does not load the weights associated with the model, only the
785
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
786
+ """
787
+
788
+
789
+ ROBERTA_INPUTS_DOCSTRING = r"""
790
+ Args:
791
+ input_ids (`torch.LongTensor` of shape `({0})`):
792
+ Indices of input sequence tokens in the vocabulary.
793
+
794
+ Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
795
+ [`PreTrainedTokenizer.__call__`] for details.
796
+
797
+ [What are input IDs?](../glossary#input-ids)
798
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
799
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
800
+
801
+ - 1 for tokens that are **not masked**,
802
+ - 0 for tokens that are **masked**.
803
+
804
+ [What are attention masks?](../glossary#attention-mask)
805
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
806
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
807
+ 1]`:
808
+
809
+ - 0 corresponds to a *sentence A* token,
810
+ - 1 corresponds to a *sentence B* token.
811
+
812
+ [What are token type IDs?](../glossary#token-type-ids)
813
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
814
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
815
+ config.max_position_embeddings - 1]`.
816
+
817
+ [What are position IDs?](../glossary#position-ids)
818
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
819
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
820
+
821
+ - 1 indicates the head is **not masked**,
822
+ - 0 indicates the head is **masked**.
823
+
824
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
825
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
826
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
827
+ model's internal embedding lookup matrix.
828
+ output_attentions (`bool`, *optional*):
829
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
830
+ tensors for more detail.
831
+ output_hidden_states (`bool`, *optional*):
832
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
833
+ more detail.
834
+ return_dict (`bool`, *optional*):
835
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
836
+ """
837
+
838
+
839
+ class RobertaModel(RobertaPreTrainedModel):
840
+ """
841
+
842
+ The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
843
+ cross-attention is added between the self-attention layers, following the architecture described in *Attention is
844
+ all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
845
+ Kaiser and Illia Polosukhin.
846
+
847
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
848
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
849
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
850
+
851
+ .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762
852
+
853
+ """
854
+
855
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
856
+
857
+ # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->Roberta
858
+ def __init__(self, config, add_pooling_layer=True):
859
+ super().__init__(config)
860
+ self.config = config
861
+
862
+ self.embeddings = RobertaEmbeddings(config)
863
+ self.encoder = RobertaEncoder(config)
864
+
865
+ self.pooler = RobertaPooler(config) if add_pooling_layer else None
866
+
867
+ # Initialize weights and apply final processing
868
+ self.post_init()
869
+
870
+ def get_input_embeddings(self):
871
+ return self.embeddings.word_embeddings
872
+
873
+ def set_input_embeddings(self, value):
874
+ self.embeddings.word_embeddings = value
875
+
876
+ def _prune_heads(self, heads_to_prune):
877
+ """
878
+ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
879
+ class PreTrainedModel
880
+ """
881
+ for layer, heads in heads_to_prune.items():
882
+ self.encoder.layer[layer].attention.prune_heads(heads)
883
+
884
+ # Copied from transformers.models.bert.modeling_bert.BertModel.forward
885
+ def forward(
886
+ self,
887
+ input_ids: Optional[torch.Tensor] = None,
888
+ attention_mask: Optional[torch.Tensor] = None,
889
+ token_type_ids: Optional[torch.Tensor] = None,
890
+ position_ids: Optional[torch.Tensor] = None,
891
+ head_mask: Optional[torch.Tensor] = None,
892
+ inputs_embeds: Optional[torch.Tensor] = None,
893
+ encoder_hidden_states: Optional[torch.Tensor] = None,
894
+ encoder_attention_mask: Optional[torch.Tensor] = None,
895
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
896
+ use_cache: Optional[bool] = None,
897
+ output_attentions: Optional[bool] = None,
898
+ output_hidden_states: Optional[bool] = None,
899
+ return_dict: Optional[bool] = None,
900
+ parser_att_mask=None,
901
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]:
902
+ r"""
903
+ encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
904
+ Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
905
+ the model is configured as a decoder.
906
+ encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
907
+ Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
908
+ the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:
909
+
910
+ - 1 for tokens that are **not masked**,
911
+ - 0 for tokens that are **masked**.
912
+ past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
913
+ Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
914
+
915
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
916
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
917
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
918
+ use_cache (`bool`, *optional*):
919
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
920
+ `past_key_values`).
921
+ """
922
+ output_attentions = (
923
+ output_attentions
924
+ if output_attentions is not None
925
+ else self.config.output_attentions
926
+ )
927
+ output_hidden_states = (
928
+ output_hidden_states
929
+ if output_hidden_states is not None
930
+ else self.config.output_hidden_states
931
+ )
932
+ return_dict = (
933
+ return_dict if return_dict is not None else self.config.use_return_dict
934
+ )
935
+
936
+ if self.config.is_decoder:
937
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
938
+ else:
939
+ use_cache = False
940
+
941
+ if input_ids is not None and inputs_embeds is not None:
942
+ raise ValueError(
943
+ "You cannot specify both input_ids and inputs_embeds at the same time"
944
+ )
945
+ elif input_ids is not None:
946
+ input_shape = input_ids.size()
947
+ elif inputs_embeds is not None:
948
+ input_shape = inputs_embeds.size()[:-1]
949
+ else:
950
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
951
+
952
+ batch_size, seq_length = input_shape
953
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
954
+
955
+ # past_key_values_length
956
+ past_key_values_length = (
957
+ past_key_values[0][0].shape[2] if past_key_values is not None else 0
958
+ )
959
+
960
+ if attention_mask is None:
961
+ attention_mask = torch.ones(
962
+ ((batch_size, seq_length + past_key_values_length)), device=device
963
+ )
964
+
965
+ if token_type_ids is None:
966
+ if hasattr(self.embeddings, "token_type_ids"):
967
+ buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
968
+ buffered_token_type_ids_expanded = buffered_token_type_ids.expand(
969
+ batch_size, seq_length
970
+ )
971
+ token_type_ids = buffered_token_type_ids_expanded
972
+ else:
973
+ token_type_ids = torch.zeros(
974
+ input_shape, dtype=torch.long, device=device
975
+ )
976
+
977
+ # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
978
+ # ourselves in which case we just need to make it broadcastable to all heads.
979
+ extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(
980
+ attention_mask, input_shape, device
981
+ )
982
+
983
+ # If a 2D or 3D attention mask is provided for the cross-attention
984
+ # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
985
+ if self.config.is_decoder and encoder_hidden_states is not None:
986
+ (
987
+ encoder_batch_size,
988
+ encoder_sequence_length,
989
+ _,
990
+ ) = encoder_hidden_states.size()
991
+ encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
992
+ if encoder_attention_mask is None:
993
+ encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
994
+ encoder_extended_attention_mask = self.invert_attention_mask(
995
+ encoder_attention_mask
996
+ )
997
+ else:
998
+ encoder_extended_attention_mask = None
999
+
1000
+ # Prepare head mask if needed
1001
+ # 1.0 in head_mask indicate we keep the head
1002
+ # attention_probs has shape bsz x n_heads x N x N
1003
+ # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
1004
+ # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
1005
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
1006
+
1007
+ embedding_output = self.embeddings(
1008
+ input_ids=input_ids,
1009
+ position_ids=position_ids,
1010
+ token_type_ids=token_type_ids,
1011
+ inputs_embeds=inputs_embeds,
1012
+ past_key_values_length=past_key_values_length,
1013
+ )
1014
+ encoder_outputs = self.encoder(
1015
+ embedding_output,
1016
+ attention_mask=extended_attention_mask,
1017
+ head_mask=head_mask,
1018
+ encoder_hidden_states=encoder_hidden_states,
1019
+ encoder_attention_mask=encoder_extended_attention_mask,
1020
+ past_key_values=past_key_values,
1021
+ use_cache=use_cache,
1022
+ output_attentions=output_attentions,
1023
+ output_hidden_states=output_hidden_states,
1024
+ return_dict=return_dict,
1025
+ parser_att_mask=parser_att_mask,
1026
+ )
1027
+ sequence_output = encoder_outputs[0]
1028
+ pooled_output = (
1029
+ self.pooler(sequence_output) if self.pooler is not None else None
1030
+ )
1031
+
1032
+ if not return_dict:
1033
+ return (sequence_output, pooled_output) + encoder_outputs[1:]
1034
+
1035
+ return BaseModelOutputWithPoolingAndCrossAttentions(
1036
+ last_hidden_state=sequence_output,
1037
+ pooler_output=pooled_output,
1038
+ past_key_values=encoder_outputs.past_key_values,
1039
+ hidden_states=encoder_outputs.hidden_states,
1040
+ attentions=encoder_outputs.attentions,
1041
+ cross_attentions=encoder_outputs.cross_attentions,
1042
+ )
1043
+
1044
+
1045
+ class StructRoberta(RobertaPreTrainedModel):
1046
+ _keys_to_ignore_on_save = [r"lm_head.decoder.weight", r"lm_head.decoder.bias"]
1047
+ _keys_to_ignore_on_load_missing = [
1048
+ r"position_ids",
1049
+ r"lm_head.decoder.weight",
1050
+ r"lm_head.decoder.bias",
1051
+ ]
1052
+ _keys_to_ignore_on_load_unexpected = [r"pooler"]
1053
+
1054
+ def __init__(self, config):
1055
+ super().__init__(config)
1056
+
1057
+ if config.is_decoder:
1058
+ logger.warning(
1059
+ "If you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for "
1060
+ "bi-directional self-attention."
1061
+ )
1062
+
1063
+
1064
+ if config.n_cntxt_layers > 0:
1065
+ config_cntxt = copy.deepcopy(config)
1066
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1067
+
1068
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1069
+
1070
+ if config.n_cntxt_layers_2 > 0:
1071
+ self.parser_layers_1 = nn.ModuleList(
1072
+ [
1073
+ nn.Sequential(
1074
+ Conv1d(config.hidden_size, config.conv_size),
1075
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1076
+ nn.Tanh(),
1077
+ )
1078
+ for i in range(int(config.n_parser_layers/2))
1079
+ ]
1080
+ )
1081
+
1082
+ self.distance_ff_1 = nn.Sequential(
1083
+ Conv1d(config.hidden_size, 2),
1084
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1085
+ nn.Tanh(),
1086
+ nn.Linear(config.hidden_size, 1),
1087
+ )
1088
+
1089
+ self.height_ff_1 = nn.Sequential(
1090
+ nn.Linear(config.hidden_size, config.hidden_size),
1091
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1092
+ nn.Tanh(),
1093
+ nn.Linear(config.hidden_size, 1),
1094
+ )
1095
+
1096
+ n_rel = len(config.relations)
1097
+ self._rel_weight_1 = nn.Parameter(
1098
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1099
+ )
1100
+ self._rel_weight_1.data.normal_(0, 0.1)
1101
+
1102
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1103
+
1104
+ config_cntxt_2 = copy.deepcopy(config)
1105
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1106
+
1107
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1108
+
1109
+
1110
+ self.parser_layers_2 = nn.ModuleList(
1111
+ [
1112
+ nn.Sequential(
1113
+ Conv1d(config.hidden_size, config.conv_size),
1114
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1115
+ nn.Tanh(),
1116
+ )
1117
+ for i in range(int(config.n_parser_layers/2))
1118
+ ]
1119
+ )
1120
+
1121
+ self.distance_ff_2 = nn.Sequential(
1122
+ Conv1d(config.hidden_size, 2),
1123
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1124
+ nn.Tanh(),
1125
+ nn.Linear(config.hidden_size, 1),
1126
+ )
1127
+
1128
+ self.height_ff_2 = nn.Sequential(
1129
+ nn.Linear(config.hidden_size, config.hidden_size),
1130
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1131
+ nn.Tanh(),
1132
+ nn.Linear(config.hidden_size, 1),
1133
+ )
1134
+
1135
+ n_rel = len(config.relations)
1136
+ self._rel_weight_2 = nn.Parameter(
1137
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1138
+ )
1139
+ self._rel_weight_2.data.normal_(0, 0.1)
1140
+
1141
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1142
+
1143
+ else:
1144
+ self.parser_layers = nn.ModuleList(
1145
+ [
1146
+ nn.Sequential(
1147
+ Conv1d(config.hidden_size, config.conv_size),
1148
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1149
+ nn.Tanh(),
1150
+ )
1151
+ for i in range(config.n_parser_layers)
1152
+ ]
1153
+ )
1154
+
1155
+ self.distance_ff = nn.Sequential(
1156
+ Conv1d(config.hidden_size, 2),
1157
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1158
+ nn.Tanh(),
1159
+ nn.Linear(config.hidden_size, 1),
1160
+ )
1161
+
1162
+ self.height_ff = nn.Sequential(
1163
+ nn.Linear(config.hidden_size, config.hidden_size),
1164
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1165
+ nn.Tanh(),
1166
+ nn.Linear(config.hidden_size, 1),
1167
+ )
1168
+
1169
+ n_rel = len(config.relations)
1170
+ self._rel_weight = nn.Parameter(
1171
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1172
+ )
1173
+ self._rel_weight.data.normal_(0, 0.1)
1174
+
1175
+ self._scaler = nn.Parameter(torch.zeros(2))
1176
+
1177
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1178
+
1179
+ if config.n_cntxt_layers > 0:
1180
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1181
+ if config.n_cntxt_layers_2 > 0:
1182
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1183
+
1184
+ self.lm_head = RobertaLMHead(config)
1185
+
1186
+ self.pad = config.pad_token_id
1187
+
1188
+ # The LM head weights require special treatment only when they are tied with the word embeddings
1189
+ self.update_keys_to_ignore(config, ["lm_head.decoder.weight"])
1190
+
1191
+ # Initialize weights and apply final processing
1192
+ self.post_init()
1193
+
1194
+ def get_output_embeddings(self):
1195
+ return self.lm_head.decoder
1196
+
1197
+ def set_output_embeddings(self, new_embeddings):
1198
+ self.lm_head.decoder = new_embeddings
1199
+
1200
+ @property
1201
+ def scaler(self):
1202
+ return self._scaler.exp()
1203
+
1204
+ @property
1205
+ def scaler_1(self):
1206
+ return self._scaler_1.exp()
1207
+
1208
+ @property
1209
+ def scaler_2(self):
1210
+ return self._scaler_2.exp()
1211
+
1212
+ @property
1213
+ def rel_weight(self):
1214
+ if self.config.weight_act == "sigmoid":
1215
+ return torch.sigmoid(self._rel_weight)
1216
+ elif self.config.weight_act == "softmax":
1217
+ return torch.softmax(self._rel_weight, dim=-1)
1218
+
1219
+ @property
1220
+ def rel_weight_1(self):
1221
+ if self.config.weight_act == "sigmoid":
1222
+ return torch.sigmoid(self._rel_weight_1)
1223
+ elif self.config.weight_act == "softmax":
1224
+ return torch.softmax(self._rel_weight_1, dim=-1)
1225
+
1226
+
1227
+ @property
1228
+ def rel_weight_2(self):
1229
+ if self.config.weight_act == "sigmoid":
1230
+ return torch.sigmoid(self._rel_weight_2)
1231
+ elif self.config.weight_act == "softmax":
1232
+ return torch.softmax(self._rel_weight_2, dim=-1)
1233
+
1234
+
1235
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1236
+ """Compute constituents from distance and height."""
1237
+
1238
+ if n_cntxt_layers>0:
1239
+ if n_cntxt_layers == 1:
1240
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1241
+ elif n_cntxt_layers == 2:
1242
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1243
+ else:
1244
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1245
+
1246
+ gamma = torch.sigmoid(-beta_logits)
1247
+ ones = torch.ones_like(gamma)
1248
+
1249
+ block_mask_left = cummin(
1250
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1251
+ )
1252
+ block_mask_left = block_mask_left - F.pad(
1253
+ block_mask_left[:, :, :-1], (1, 0), value=0
1254
+ )
1255
+ block_mask_left.tril_(0)
1256
+
1257
+ block_mask_right = cummin(
1258
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1259
+ )
1260
+ block_mask_right = block_mask_right - F.pad(
1261
+ block_mask_right[:, :, 1:], (0, 1), value=0
1262
+ )
1263
+ block_mask_right.triu_(0)
1264
+
1265
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1266
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1267
+ block_mask_right, reverse=True
1268
+ ).triu(1)
1269
+
1270
+ return block_p, block
1271
+
1272
+ def compute_head(self, height, n_cntxt_layers=0):
1273
+ """Estimate head for each constituent."""
1274
+
1275
+ _, length = height.size()
1276
+ if n_cntxt_layers>0:
1277
+ if n_cntxt_layers == 1:
1278
+ head_logits = height * self.scaler_1[1]
1279
+ elif n_cntxt_layers == 2:
1280
+ head_logits = height * self.scaler_2[1]
1281
+ else:
1282
+ head_logits = height * self.scaler[1]
1283
+ index = torch.arange(length, device=height.device)
1284
+
1285
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1286
+ index[None, None, :] <= index[None, :, None]
1287
+ )
1288
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1289
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1290
+
1291
+ head_p = torch.softmax(head_logits, dim=-1)
1292
+
1293
+ return head_p
1294
+
1295
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1296
+ """Parse input sentence.
1297
+
1298
+ Args:
1299
+ x: input tokens (required).
1300
+ pos: position for each token (optional).
1301
+ Returns:
1302
+ distance: syntactic distance
1303
+ height: syntactic height
1304
+ """
1305
+
1306
+ mask = x != self.pad
1307
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1308
+
1309
+ if embs is None:
1310
+ h = self.roberta.embeddings(x)
1311
+ else:
1312
+ h = embs
1313
+
1314
+ if n_cntxt_layers > 0:
1315
+ if n_cntxt_layers == 1:
1316
+ parser_layers = self.parser_layers_1
1317
+ height_ff = self.height_ff_1
1318
+ distance_ff = self.distance_ff_1
1319
+ elif n_cntxt_layers == 2:
1320
+ parser_layers = self.parser_layers_2
1321
+ height_ff = self.height_ff_2
1322
+ distance_ff = self.distance_ff_2
1323
+ for i in range(int(self.config.n_parser_layers/2)):
1324
+ h = h.masked_fill(~mask[:, :, None], 0)
1325
+ h = parser_layers[i](h)
1326
+
1327
+ height = height_ff(h).squeeze(-1)
1328
+ height.masked_fill_(~mask, -1e9)
1329
+
1330
+ distance = distance_ff(h).squeeze(-1)
1331
+ distance.masked_fill_(~mask_shifted, 1e9)
1332
+
1333
+ # Calbrating the distance and height to the same level
1334
+ length = distance.size(1)
1335
+ height_max = height[:, None, :].expand(-1, length, -1)
1336
+ height_max = torch.cummax(
1337
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1338
+ )[0].triu(0)
1339
+
1340
+ margin_left = torch.relu(
1341
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1342
+ )
1343
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1344
+ margin = torch.where(
1345
+ margin_left > margin_right, margin_right, margin_left
1346
+ ).triu(0)
1347
+
1348
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1349
+ margin.masked_fill_(~margin_mask, 0)
1350
+ margin = margin.max()
1351
+
1352
+ distance = distance - margin
1353
+ else:
1354
+ for i in range(self.config.n_parser_layers):
1355
+ h = h.masked_fill(~mask[:, :, None], 0)
1356
+ h = self.parser_layers[i](h)
1357
+
1358
+ height = self.height_ff(h).squeeze(-1)
1359
+ height.masked_fill_(~mask, -1e9)
1360
+
1361
+ distance = self.distance_ff(h).squeeze(-1)
1362
+ distance.masked_fill_(~mask_shifted, 1e9)
1363
+
1364
+ # Calbrating the distance and height to the same level
1365
+ length = distance.size(1)
1366
+ height_max = height[:, None, :].expand(-1, length, -1)
1367
+ height_max = torch.cummax(
1368
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1369
+ )[0].triu(0)
1370
+
1371
+ margin_left = torch.relu(
1372
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1373
+ )
1374
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1375
+ margin = torch.where(
1376
+ margin_left > margin_right, margin_right, margin_left
1377
+ ).triu(0)
1378
+
1379
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1380
+ margin.masked_fill_(~margin_mask, 0)
1381
+ margin = margin.max()
1382
+
1383
+ distance = distance - margin
1384
+
1385
+ return distance, height
1386
+
1387
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1388
+ """Compute head and cibling distribution for each token."""
1389
+
1390
+ bsz, length = x.size()
1391
+
1392
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1393
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1394
+
1395
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1396
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1397
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1398
+ head = head.masked_fill(eye, 0)
1399
+ child = head.transpose(1, 2)
1400
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1401
+
1402
+ rel_list = []
1403
+ if "head" in self.config.relations:
1404
+ rel_list.append(head)
1405
+ if "child" in self.config.relations:
1406
+ rel_list.append(child)
1407
+ if "cibling" in self.config.relations:
1408
+ rel_list.append(cibling)
1409
+
1410
+ rel = torch.stack(rel_list, dim=1)
1411
+
1412
+ if n_cntxt_layers > 0:
1413
+ if n_cntxt_layers == 1:
1414
+ rel_weight = self.rel_weight_1
1415
+ elif n_cntxt_layers == 2:
1416
+ rel_weight = self.rel_weight_2
1417
+ else:
1418
+ rel_weight = self.rel_weight
1419
+
1420
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1421
+
1422
+ if n_cntxt_layers == 1:
1423
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1424
+ else:
1425
+ num_layers = self.roberta.config.num_hidden_layers
1426
+
1427
+ att_mask = dep.reshape(
1428
+ num_layers,
1429
+ bsz,
1430
+ self.config.num_attention_heads,
1431
+ length,
1432
+ length,
1433
+ )
1434
+
1435
+ return att_mask, cibling, head, block
1436
+
1437
+ def forward(
1438
+ self,
1439
+ input_ids: Optional[torch.LongTensor] = None,
1440
+ attention_mask: Optional[torch.FloatTensor] = None,
1441
+ token_type_ids: Optional[torch.LongTensor] = None,
1442
+ position_ids: Optional[torch.LongTensor] = None,
1443
+ head_mask: Optional[torch.FloatTensor] = None,
1444
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1445
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
1446
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
1447
+ labels: Optional[torch.LongTensor] = None,
1448
+ output_attentions: Optional[bool] = None,
1449
+ output_hidden_states: Optional[bool] = None,
1450
+ return_dict: Optional[bool] = None,
1451
+ ) -> Union[Tuple, MaskedLMOutput]:
1452
+ r"""
1453
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1454
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
1455
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
1456
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
1457
+ kwargs (`Dict[str, any]`, optional, defaults to *{}*):
1458
+ Used to hide legacy arguments that have been deprecated.
1459
+ """
1460
+ return_dict = (
1461
+ return_dict if return_dict is not None else self.config.use_return_dict
1462
+ )
1463
+
1464
+
1465
+ if self.config.n_cntxt_layers > 0:
1466
+ cntxt_outputs = self.cntxt_layers(
1467
+ input_ids,
1468
+ attention_mask=attention_mask,
1469
+ token_type_ids=token_type_ids,
1470
+ position_ids=position_ids,
1471
+ head_mask=head_mask,
1472
+ inputs_embeds=inputs_embeds,
1473
+ encoder_hidden_states=encoder_hidden_states,
1474
+ encoder_attention_mask=encoder_attention_mask,
1475
+ output_attentions=output_attentions,
1476
+ output_hidden_states=output_hidden_states,
1477
+ return_dict=return_dict)
1478
+
1479
+
1480
+ if self.config.n_cntxt_layers_2 > 0:
1481
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1482
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1483
+
1484
+ cntxt_outputs_2 = self.cntxt_layers_2(
1485
+ input_ids,
1486
+ attention_mask=attention_mask,
1487
+ token_type_ids=token_type_ids,
1488
+ position_ids=position_ids,
1489
+ head_mask=head_mask,
1490
+ inputs_embeds=inputs_embeds,
1491
+ encoder_hidden_states=encoder_hidden_states,
1492
+ encoder_attention_mask=encoder_attention_mask,
1493
+ output_attentions=output_attentions,
1494
+ output_hidden_states=output_hidden_states,
1495
+ return_dict=return_dict,
1496
+ parser_att_mask=att_mask_1)
1497
+
1498
+ sequence_output = cntxt_outputs_2[0]
1499
+
1500
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
1501
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
1502
+
1503
+ elif self.config.n_cntxt_layers > 0:
1504
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
1505
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1506
+ else:
1507
+ distance, height = self.parse(input_ids)
1508
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
1509
+
1510
+ outputs = self.roberta(
1511
+ input_ids,
1512
+ attention_mask=attention_mask,
1513
+ token_type_ids=token_type_ids,
1514
+ position_ids=position_ids,
1515
+ head_mask=head_mask,
1516
+ inputs_embeds=inputs_embeds,
1517
+ encoder_hidden_states=encoder_hidden_states,
1518
+ encoder_attention_mask=encoder_attention_mask,
1519
+ output_attentions=output_attentions,
1520
+ output_hidden_states=output_hidden_states,
1521
+ return_dict=return_dict,
1522
+ parser_att_mask=att_mask,
1523
+ )
1524
+ sequence_output = outputs[0]
1525
+ prediction_scores = self.lm_head(sequence_output)
1526
+
1527
+ masked_lm_loss = None
1528
+ if labels is not None:
1529
+ loss_fct = CrossEntropyLoss()
1530
+ masked_lm_loss = loss_fct(
1531
+ prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)
1532
+ )
1533
+
1534
+ if not return_dict:
1535
+ output = (prediction_scores,) + outputs[2:]
1536
+ return (
1537
+ ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
1538
+ )
1539
+
1540
+ return MaskedLMOutput(
1541
+ loss=masked_lm_loss,
1542
+ logits=prediction_scores,
1543
+ hidden_states=outputs.hidden_states,
1544
+ attentions=outputs.attentions,
1545
+ )
1546
+
1547
+
1548
+ class RobertaLMHead(nn.Module):
1549
+ """Roberta Head for masked language modeling."""
1550
+
1551
+ def __init__(self, config):
1552
+ super().__init__()
1553
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
1554
+ self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
1555
+
1556
+ self.decoder = nn.Linear(config.hidden_size, config.vocab_size)
1557
+ self.bias = nn.Parameter(torch.zeros(config.vocab_size))
1558
+ self.decoder.bias = self.bias
1559
+
1560
+ def forward(self, features, **kwargs):
1561
+ x = self.dense(features)
1562
+ x = gelu(x)
1563
+ x = self.layer_norm(x)
1564
+
1565
+ # project back to size of vocabulary with bias
1566
+ x = self.decoder(x)
1567
+
1568
+ return x
1569
+
1570
+ def _tie_weights(self):
1571
+ # To tie those two weights if they get disconnected (on TPU or when the bias is resized)
1572
+ self.bias = self.decoder.bias
1573
+
1574
+
1575
+ class StructRobertaForSequenceClassification(RobertaPreTrainedModel):
1576
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
1577
+
1578
+ def __init__(self, config):
1579
+ super().__init__(config)
1580
+ self.num_labels = config.num_labels
1581
+ self.config = config
1582
+
1583
+ if config.n_cntxt_layers > 0:
1584
+ config_cntxt = copy.deepcopy(config)
1585
+ config_cntxt.num_hidden_layers = config.n_cntxt_layers
1586
+
1587
+ self.cntxt_layers = RobertaModel(config_cntxt, add_pooling_layer=False)
1588
+
1589
+ if config.n_cntxt_layers_2 > 0:
1590
+ self.parser_layers_1 = nn.ModuleList(
1591
+ [
1592
+ nn.Sequential(
1593
+ Conv1d(config.hidden_size, config.conv_size),
1594
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1595
+ nn.Tanh(),
1596
+ )
1597
+ for i in range(int(config.n_parser_layers/2))
1598
+ ]
1599
+ )
1600
+
1601
+ self.distance_ff_1 = nn.Sequential(
1602
+ Conv1d(config.hidden_size, 2),
1603
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1604
+ nn.Tanh(),
1605
+ nn.Linear(config.hidden_size, 1),
1606
+ )
1607
+
1608
+ self.height_ff_1 = nn.Sequential(
1609
+ nn.Linear(config.hidden_size, config.hidden_size),
1610
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1611
+ nn.Tanh(),
1612
+ nn.Linear(config.hidden_size, 1),
1613
+ )
1614
+
1615
+ n_rel = len(config.relations)
1616
+ self._rel_weight_1 = nn.Parameter(
1617
+ torch.zeros((config.n_cntxt_layers_2, config.num_attention_heads, n_rel))
1618
+ )
1619
+ self._rel_weight_1.data.normal_(0, 0.1)
1620
+
1621
+ self._scaler_1 = nn.Parameter(torch.zeros(2))
1622
+
1623
+ config_cntxt_2 = copy.deepcopy(config)
1624
+ config_cntxt_2.num_hidden_layers = config.n_cntxt_layers_2
1625
+
1626
+ self.cntxt_layers_2 = RobertaModel(config_cntxt_2, add_pooling_layer=False)
1627
+
1628
+
1629
+ self.parser_layers_2 = nn.ModuleList(
1630
+ [
1631
+ nn.Sequential(
1632
+ Conv1d(config.hidden_size, config.conv_size),
1633
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1634
+ nn.Tanh(),
1635
+ )
1636
+ for i in range(int(config.n_parser_layers/2))
1637
+ ]
1638
+ )
1639
+
1640
+ self.distance_ff_2 = nn.Sequential(
1641
+ Conv1d(config.hidden_size, 2),
1642
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1643
+ nn.Tanh(),
1644
+ nn.Linear(config.hidden_size, 1),
1645
+ )
1646
+
1647
+ self.height_ff_2 = nn.Sequential(
1648
+ nn.Linear(config.hidden_size, config.hidden_size),
1649
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1650
+ nn.Tanh(),
1651
+ nn.Linear(config.hidden_size, 1),
1652
+ )
1653
+
1654
+ n_rel = len(config.relations)
1655
+ self._rel_weight_2 = nn.Parameter(
1656
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1657
+ )
1658
+ self._rel_weight_2.data.normal_(0, 0.1)
1659
+
1660
+ self._scaler_2 = nn.Parameter(torch.zeros(2))
1661
+
1662
+ else:
1663
+ self.parser_layers = nn.ModuleList(
1664
+ [
1665
+ nn.Sequential(
1666
+ Conv1d(config.hidden_size, config.conv_size),
1667
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1668
+ nn.Tanh(),
1669
+ )
1670
+ for i in range(config.n_parser_layers)
1671
+ ]
1672
+ )
1673
+
1674
+ self.distance_ff = nn.Sequential(
1675
+ Conv1d(config.hidden_size, 2),
1676
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1677
+ nn.Tanh(),
1678
+ nn.Linear(config.hidden_size, 1),
1679
+ )
1680
+
1681
+ self.height_ff = nn.Sequential(
1682
+ nn.Linear(config.hidden_size, config.hidden_size),
1683
+ nn.LayerNorm(config.hidden_size, elementwise_affine=False),
1684
+ nn.Tanh(),
1685
+ nn.Linear(config.hidden_size, 1),
1686
+ )
1687
+
1688
+ n_rel = len(config.relations)
1689
+ self._rel_weight = nn.Parameter(
1690
+ torch.zeros((config.num_hidden_layers, config.num_attention_heads, n_rel))
1691
+ )
1692
+ self._rel_weight.data.normal_(0, 0.1)
1693
+
1694
+ self._scaler = nn.Parameter(torch.zeros(2))
1695
+
1696
+ self.roberta = RobertaModel(config, add_pooling_layer=False)
1697
+
1698
+ if config.n_cntxt_layers > 0:
1699
+ self.cntxt_layers.embeddings = self.roberta.embeddings
1700
+ if config.n_cntxt_layers_2 > 0:
1701
+ self.cntxt_layers_2.embeddings = self.roberta.embeddings
1702
+
1703
+
1704
+ self.pad = config.pad_token_id
1705
+ self.classifier = RobertaClassificationHead(config)
1706
+
1707
+ # Initialize weights and apply final processing
1708
+ self.post_init()
1709
+
1710
+
1711
+ @property
1712
+ def scaler(self):
1713
+ return self._scaler.exp()
1714
+
1715
+ @property
1716
+ def scaler_1(self):
1717
+ return self._scaler_1.exp()
1718
+
1719
+ @property
1720
+ def scaler_2(self):
1721
+ return self._scaler_2.exp()
1722
+
1723
+ @property
1724
+ def rel_weight(self):
1725
+ if self.config.weight_act == "sigmoid":
1726
+ return torch.sigmoid(self._rel_weight)
1727
+ elif self.config.weight_act == "softmax":
1728
+ return torch.softmax(self._rel_weight, dim=-1)
1729
+
1730
+ @property
1731
+ def rel_weight_1(self):
1732
+ if self.config.weight_act == "sigmoid":
1733
+ return torch.sigmoid(self._rel_weight_1)
1734
+ elif self.config.weight_act == "softmax":
1735
+ return torch.softmax(self._rel_weight_1, dim=-1)
1736
+
1737
+
1738
+ @property
1739
+ def rel_weight_2(self):
1740
+ if self.config.weight_act == "sigmoid":
1741
+ return torch.sigmoid(self._rel_weight_2)
1742
+ elif self.config.weight_act == "softmax":
1743
+ return torch.softmax(self._rel_weight_2, dim=-1)
1744
+
1745
+
1746
+ def compute_block(self, distance, height, n_cntxt_layers=0):
1747
+ """Compute constituents from distance and height."""
1748
+
1749
+ if n_cntxt_layers>0:
1750
+ if n_cntxt_layers == 1:
1751
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_1[0]
1752
+ elif n_cntxt_layers == 2:
1753
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler_2[0]
1754
+ else:
1755
+ beta_logits = (distance[:, None, :] - height[:, :, None]) * self.scaler[0]
1756
+
1757
+ gamma = torch.sigmoid(-beta_logits)
1758
+ ones = torch.ones_like(gamma)
1759
+
1760
+ block_mask_left = cummin(
1761
+ gamma.tril(-1) + ones.triu(0), reverse=True, max_value=1
1762
+ )
1763
+ block_mask_left = block_mask_left - F.pad(
1764
+ block_mask_left[:, :, :-1], (1, 0), value=0
1765
+ )
1766
+ block_mask_left.tril_(0)
1767
+
1768
+ block_mask_right = cummin(
1769
+ gamma.triu(0) + ones.tril(-1), exclusive=True, max_value=1
1770
+ )
1771
+ block_mask_right = block_mask_right - F.pad(
1772
+ block_mask_right[:, :, 1:], (0, 1), value=0
1773
+ )
1774
+ block_mask_right.triu_(0)
1775
+
1776
+ block_p = block_mask_left[:, :, :, None] * block_mask_right[:, :, None, :]
1777
+ block = cumsum(block_mask_left).tril(0) + cumsum(
1778
+ block_mask_right, reverse=True
1779
+ ).triu(1)
1780
+
1781
+ return block_p, block
1782
+
1783
+ def compute_head(self, height, n_cntxt_layers=0):
1784
+ """Estimate head for each constituent."""
1785
+
1786
+ _, length = height.size()
1787
+ if n_cntxt_layers>0:
1788
+ if n_cntxt_layers == 1:
1789
+ head_logits = height * self.scaler_1[1]
1790
+ elif n_cntxt_layers == 2:
1791
+ head_logits = height * self.scaler_2[1]
1792
+ else:
1793
+ head_logits = height * self.scaler[1]
1794
+ index = torch.arange(length, device=height.device)
1795
+
1796
+ mask = (index[:, None, None] <= index[None, None, :]) * (
1797
+ index[None, None, :] <= index[None, :, None]
1798
+ )
1799
+ head_logits = head_logits[:, None, None, :].repeat(1, length, length, 1)
1800
+ head_logits.masked_fill_(~mask[None, :, :, :], -1e9)
1801
+
1802
+ head_p = torch.softmax(head_logits, dim=-1)
1803
+
1804
+ return head_p
1805
+
1806
+ def parse(self, x, embs=None, n_cntxt_layers=0):
1807
+ """Parse input sentence.
1808
+
1809
+ Args:
1810
+ x: input tokens (required).
1811
+ pos: position for each token (optional).
1812
+ Returns:
1813
+ distance: syntactic distance
1814
+ height: syntactic height
1815
+ """
1816
+
1817
+ mask = x != self.pad
1818
+ mask_shifted = F.pad(mask[:, 1:], (0, 1), value=0)
1819
+
1820
+ if embs is None:
1821
+ h = self.roberta.embeddings(x)
1822
+ else:
1823
+ h = embs
1824
+
1825
+ if n_cntxt_layers > 0:
1826
+ if n_cntxt_layers == 1:
1827
+ parser_layers = self.parser_layers_1
1828
+ height_ff = self.height_ff_1
1829
+ distance_ff = self.distance_ff_1
1830
+ elif n_cntxt_layers == 2:
1831
+ parser_layers = self.parser_layers_2
1832
+ height_ff = self.height_ff_2
1833
+ distance_ff = self.distance_ff_2
1834
+ for i in range(int(self.config.n_parser_layers/2)):
1835
+ h = h.masked_fill(~mask[:, :, None], 0)
1836
+ h = parser_layers[i](h)
1837
+
1838
+ height = height_ff(h).squeeze(-1)
1839
+ height.masked_fill_(~mask, -1e9)
1840
+
1841
+ distance = distance_ff(h).squeeze(-1)
1842
+ distance.masked_fill_(~mask_shifted, 1e9)
1843
+
1844
+ # Calbrating the distance and height to the same level
1845
+ length = distance.size(1)
1846
+ height_max = height[:, None, :].expand(-1, length, -1)
1847
+ height_max = torch.cummax(
1848
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1849
+ )[0].triu(0)
1850
+
1851
+ margin_left = torch.relu(
1852
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1853
+ )
1854
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1855
+ margin = torch.where(
1856
+ margin_left > margin_right, margin_right, margin_left
1857
+ ).triu(0)
1858
+
1859
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1860
+ margin.masked_fill_(~margin_mask, 0)
1861
+ margin = margin.max()
1862
+
1863
+ distance = distance - margin
1864
+ else:
1865
+ for i in range(self.config.n_parser_layers):
1866
+ h = h.masked_fill(~mask[:, :, None], 0)
1867
+ h = self.parser_layers[i](h)
1868
+
1869
+ height = self.height_ff(h).squeeze(-1)
1870
+ height.masked_fill_(~mask, -1e9)
1871
+
1872
+ distance = self.distance_ff(h).squeeze(-1)
1873
+ distance.masked_fill_(~mask_shifted, 1e9)
1874
+
1875
+ # Calbrating the distance and height to the same level
1876
+ length = distance.size(1)
1877
+ height_max = height[:, None, :].expand(-1, length, -1)
1878
+ height_max = torch.cummax(
1879
+ height_max.triu(0) - torch.ones_like(height_max).tril(-1) * 1e9, dim=-1
1880
+ )[0].triu(0)
1881
+
1882
+ margin_left = torch.relu(
1883
+ F.pad(distance[:, :-1, None], (0, 0, 1, 0), value=1e9) - height_max
1884
+ )
1885
+ margin_right = torch.relu(distance[:, None, :] - height_max)
1886
+ margin = torch.where(
1887
+ margin_left > margin_right, margin_right, margin_left
1888
+ ).triu(0)
1889
+
1890
+ margin_mask = torch.stack([mask_shifted] + [mask] * (length - 1), dim=1)
1891
+ margin.masked_fill_(~margin_mask, 0)
1892
+ margin = margin.max()
1893
+
1894
+ distance = distance - margin
1895
+
1896
+ return distance, height
1897
+
1898
+ def generate_mask(self, x, distance, height, n_cntxt_layers=0):
1899
+ """Compute head and cibling distribution for each token."""
1900
+
1901
+ bsz, length = x.size()
1902
+
1903
+ eye = torch.eye(length, device=x.device, dtype=torch.bool)
1904
+ eye = eye[None, :, :].expand((bsz, -1, -1))
1905
+
1906
+ block_p, block = self.compute_block(distance, height, n_cntxt_layers=n_cntxt_layers)
1907
+ head_p = self.compute_head(height, n_cntxt_layers=n_cntxt_layers)
1908
+ head = torch.einsum("blij,bijh->blh", block_p, head_p)
1909
+ head = head.masked_fill(eye, 0)
1910
+ child = head.transpose(1, 2)
1911
+ cibling = torch.bmm(head, child).masked_fill(eye, 0)
1912
+
1913
+ rel_list = []
1914
+ if "head" in self.config.relations:
1915
+ rel_list.append(head)
1916
+ if "child" in self.config.relations:
1917
+ rel_list.append(child)
1918
+ if "cibling" in self.config.relations:
1919
+ rel_list.append(cibling)
1920
+
1921
+ rel = torch.stack(rel_list, dim=1)
1922
+
1923
+ if n_cntxt_layers > 0:
1924
+ if n_cntxt_layers == 1:
1925
+ rel_weight = self.rel_weight_1
1926
+ elif n_cntxt_layers == 2:
1927
+ rel_weight = self.rel_weight_2
1928
+ else:
1929
+ rel_weight = self.rel_weight
1930
+
1931
+ dep = torch.einsum("lhr,brij->lbhij", rel_weight, rel)
1932
+
1933
+ if n_cntxt_layers == 1:
1934
+ num_layers = self.cntxt_layers_2.config.num_hidden_layers
1935
+ else:
1936
+ num_layers = self.roberta.config.num_hidden_layers
1937
+
1938
+ att_mask = dep.reshape(
1939
+ num_layers,
1940
+ bsz,
1941
+ self.config.num_attention_heads,
1942
+ length,
1943
+ length,
1944
+ )
1945
+
1946
+ return att_mask, cibling, head, block
1947
+
1948
+ def forward(
1949
+ self,
1950
+ input_ids: Optional[torch.LongTensor] = None,
1951
+ attention_mask: Optional[torch.FloatTensor] = None,
1952
+ token_type_ids: Optional[torch.LongTensor] = None,
1953
+ position_ids: Optional[torch.LongTensor] = None,
1954
+ head_mask: Optional[torch.FloatTensor] = None,
1955
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1956
+ labels: Optional[torch.LongTensor] = None,
1957
+ output_attentions: Optional[bool] = None,
1958
+ output_hidden_states: Optional[bool] = None,
1959
+ return_dict: Optional[bool] = None,
1960
+ ) -> Union[Tuple, SequenceClassifierOutput]:
1961
+ r"""
1962
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1963
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1964
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1965
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1966
+ """
1967
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1968
+
1969
+ if self.config.n_cntxt_layers > 0:
1970
+ cntxt_outputs = self.cntxt_layers(
1971
+ input_ids,
1972
+ attention_mask=attention_mask,
1973
+ token_type_ids=token_type_ids,
1974
+ position_ids=position_ids,
1975
+ head_mask=head_mask,
1976
+ inputs_embeds=inputs_embeds,
1977
+ output_attentions=output_attentions,
1978
+ output_hidden_states=output_hidden_states,
1979
+ return_dict=return_dict)
1980
+
1981
+
1982
+ if self.config.n_cntxt_layers_2 > 0:
1983
+ distance_1, height_1 = self.parse(input_ids, cntxt_outputs[0], n_cntxt_layers=1)
1984
+ att_mask_1, _, _, _ = self.generate_mask(input_ids, distance_1, height_1, n_cntxt_layers=1)
1985
+
1986
+ cntxt_outputs_2 = self.cntxt_layers_2(
1987
+ input_ids,
1988
+ attention_mask=attention_mask,
1989
+ token_type_ids=token_type_ids,
1990
+ position_ids=position_ids,
1991
+ head_mask=head_mask,
1992
+ inputs_embeds=inputs_embeds,
1993
+ output_attentions=output_attentions,
1994
+ output_hidden_states=output_hidden_states,
1995
+ return_dict=return_dict,
1996
+ parser_att_mask=att_mask_1)
1997
+
1998
+ sequence_output = cntxt_outputs_2[0]
1999
+
2000
+ distance_2, height_2 = self.parse(input_ids, sequence_output[0], n_cntxt_layers=2)
2001
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance_2, height_2, n_cntxt_layers=2)
2002
+
2003
+ elif self.config.n_cntxt_layers > 0:
2004
+ distance, height = self.parse(input_ids, cntxt_outputs[0])
2005
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2006
+ else:
2007
+ distance, height = self.parse(input_ids)
2008
+ att_mask, _, _, _ = self.generate_mask(input_ids, distance, height)
2009
+
2010
+ outputs = self.roberta(
2011
+ input_ids,
2012
+ attention_mask=attention_mask,
2013
+ token_type_ids=token_type_ids,
2014
+ position_ids=position_ids,
2015
+ head_mask=head_mask,
2016
+ inputs_embeds=inputs_embeds,
2017
+ output_attentions=output_attentions,
2018
+ output_hidden_states=output_hidden_states,
2019
+ return_dict=return_dict,
2020
+ parser_att_mask=att_mask,
2021
+ )
2022
+ sequence_output = outputs[0]
2023
+ logits = self.classifier(sequence_output)
2024
+
2025
+ loss = None
2026
+ if labels is not None:
2027
+ if self.config.problem_type is None:
2028
+ if self.num_labels == 1:
2029
+ self.config.problem_type = "regression"
2030
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
2031
+ self.config.problem_type = "single_label_classification"
2032
+ else:
2033
+ self.config.problem_type = "multi_label_classification"
2034
+
2035
+ if self.config.problem_type == "regression":
2036
+ loss_fct = MSELoss()
2037
+ if self.num_labels == 1:
2038
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
2039
+ else:
2040
+ loss = loss_fct(logits, labels)
2041
+ elif self.config.problem_type == "single_label_classification":
2042
+ loss_fct = CrossEntropyLoss()
2043
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
2044
+ elif self.config.problem_type == "multi_label_classification":
2045
+ loss_fct = BCEWithLogitsLoss()
2046
+ loss = loss_fct(logits, labels)
2047
+
2048
+ if not return_dict:
2049
+ output = (logits,) + outputs[2:]
2050
+ return ((loss,) + output) if loss is not None else output
2051
+
2052
+ return SequenceClassifierOutput(
2053
+ loss=loss,
2054
+ logits=logits,
2055
+ hidden_states=outputs.hidden_states,
2056
+ attentions=outputs.attentions,
2057
+ )
2058
+
2059
+
2060
+ class RobertaClassificationHead(nn.Module):
2061
+ """Head for sentence-level classification tasks."""
2062
+
2063
+ def __init__(self, config):
2064
+ super().__init__()
2065
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
2066
+ classifier_dropout = (
2067
+ config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
2068
+ )
2069
+ self.dropout = nn.Dropout(classifier_dropout)
2070
+ self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
2071
+
2072
+ def forward(self, features, **kwargs):
2073
+ x = features[:, 0, :] # take <s> token (equiv. to [CLS])
2074
+ x = self.dropout(x)
2075
+ x = self.dense(x)
2076
+ x = torch.tanh(x)
2077
+ x = self.dropout(x)
2078
+ x = self.out_proj(x)
2079
+ return x
2080
+
2081
+
2082
+ def create_position_ids_from_input_ids(
2083
+ input_ids, padding_idx, past_key_values_length=0
2084
+ ):
2085
+ """
2086
+ Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
2087
+ are ignored. This is modified from fairseq's `utils.make_positions`.
2088
+
2089
+ Args:
2090
+ x: torch.Tensor x:
2091
+
2092
+ Returns: torch.Tensor
2093
+ """
2094
+ # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
2095
+ mask = input_ids.ne(padding_idx).int()
2096
+ incremental_indices = (
2097
+ torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length
2098
+ ) * mask
2099
+ return incremental_indices.long() + padding_idx
2100
+
2101
+
2102
+ def cumprod(x, reverse=False, exclusive=False):
2103
+ """cumulative product."""
2104
+ if reverse:
2105
+ x = x.flip([-1])
2106
+
2107
+ if exclusive:
2108
+ x = F.pad(x[:, :, :-1], (1, 0), value=1)
2109
+
2110
+ cx = x.cumprod(-1)
2111
+
2112
+ if reverse:
2113
+ cx = cx.flip([-1])
2114
+ return cx
2115
+
2116
+
2117
+ def cumsum(x, reverse=False, exclusive=False):
2118
+ """cumulative sum."""
2119
+ bsz, _, length = x.size()
2120
+ device = x.device
2121
+ if reverse:
2122
+ if exclusive:
2123
+ w = torch.ones([bsz, length, length], device=device).tril(-1)
2124
+ else:
2125
+ w = torch.ones([bsz, length, length], device=device).tril(0)
2126
+ cx = torch.bmm(x, w)
2127
+ else:
2128
+ if exclusive:
2129
+ w = torch.ones([bsz, length, length], device=device).triu(1)
2130
+ else:
2131
+ w = torch.ones([bsz, length, length], device=device).triu(0)
2132
+ cx = torch.bmm(x, w)
2133
+ return cx
2134
+
2135
+
2136
+ def cummin(x, reverse=False, exclusive=False, max_value=1e9):
2137
+ """cumulative min."""
2138
+ if reverse:
2139
+ if exclusive:
2140
+ x = F.pad(x[:, :, 1:], (0, 1), value=max_value)
2141
+ x = x.flip([-1]).cummin(-1)[0].flip([-1])
2142
+ else:
2143
+ if exclusive:
2144
+ x = F.pad(x[:, :, :-1], (1, 0), value=max_value)
2145
+ x = x.cummin(-1)[0]
2146
+ return x
finetune/control_raising_lexical_content_the/predict_results.txt ADDED
The diff for this file is too large to render. See raw diff
 
finetune/control_raising_lexical_content_the/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e84a5bd04d8b06a56ba5f9a1c12bc0b94dc5870896cd7637ae49910fabf08ec1
3
+ size 534594825
finetune/control_raising_lexical_content_the/special_tokens_map.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "mask_token": {
6
+ "content": "<mask>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "pad_token": "<pad>",
13
+ "sep_token": "</s>",
14
+ "unk_token": "<unk>"
15
+ }
finetune/control_raising_lexical_content_the/tokenizer_config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": {
4
+ "__type": "AddedToken",
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ },
11
+ "cls_token": {
12
+ "__type": "AddedToken",
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false
26
+ },
27
+ "errors": "replace",
28
+ "mask_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<mask>",
31
+ "lstrip": true,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false
35
+ },
36
+ "model_max_length": 512,
37
+ "name_or_path": "final_models/glue_models/structroberta_s2_final/",
38
+ "pad_token": {
39
+ "__type": "AddedToken",
40
+ "content": "<pad>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": false
45
+ },
46
+ "sep_token": {
47
+ "__type": "AddedToken",
48
+ "content": "</s>",
49
+ "lstrip": false,
50
+ "normalized": true,
51
+ "rstrip": false,
52
+ "single_word": false
53
+ },
54
+ "special_tokens_map_file": null,
55
+ "tokenizer_class": "RobertaTokenizer",
56
+ "trim_offsets": true,
57
+ "unk_token": {
58
+ "__type": "AddedToken",
59
+ "content": "<unk>",
60
+ "lstrip": false,
61
+ "normalized": true,
62
+ "rstrip": false,
63
+ "single_word": false
64
+ }
65
+ }
finetune/control_raising_lexical_content_the/train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "train_loss": 0.011623076279239173,
4
+ "train_runtime": 335.2215,
5
+ "train_samples": 6816,
6
+ "train_samples_per_second": 203.328,
7
+ "train_steps_per_second": 1.7
8
+ }