Upload folder using huggingface_hub

README.md

@@ -0,0 +1,89 @@
+---
+license: mit
+base_model: microsoft/deberta-v3-base
+tags:
+- generated_from_trainer
+metrics:
+- precision
+- recall
+- f1
+- accuracy
+model-index:
+- name: our_data
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# our_data
+
+This model is a fine-tuned version of [microsoft/deberta-v3-base](https://huggingface.co/microsoft/deberta-v3-base) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.5468
+- Precision: 0.4730
+- Recall: 0.5569
+- F1: 0.5115
+- Accuracy: 0.7401
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 10.0
+
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss | Precision | Recall | F1     | Accuracy |
+|:-------------:|:-----:|:-----:|:---------------:|:---------:|:------:|:------:|:--------:|
+| 1.7886        | 0.4   | 500   | 1.5075          | 0.1842    | 0.2103 | 0.1964 | 0.6169   |
+| 1.3644        | 0.81  | 1000  | 1.3342          | 0.2364    | 0.3056 | 0.2666 | 0.6492   |
+| 1.1181        | 1.21  | 1500  | 1.2655          | 0.2959    | 0.3585 | 0.3242 | 0.6812   |
+| 0.9833        | 1.61  | 2000  | 1.2368          | 0.2941    | 0.3902 | 0.3354 | 0.6778   |
+| 0.9036        | 2.01  | 2500  | 1.2682          | 0.3551    | 0.4021 | 0.3772 | 0.7023   |
+| 0.7102        | 2.42  | 3000  | 1.2176          | 0.3668    | 0.4590 | 0.4078 | 0.7159   |
+| 0.6868        | 2.82  | 3500  | 1.2170          | 0.3794    | 0.4683 | 0.4192 | 0.7147   |
+| 0.5671        | 3.22  | 4000  | 1.2603          | 0.3951    | 0.4881 | 0.4367 | 0.7259   |
+| 0.4878        | 3.63  | 4500  | 1.2460          | 0.3925    | 0.5093 | 0.4433 | 0.7333   |
+| 0.4942        | 4.03  | 5000  | 1.3147          | 0.4047    | 0.4802 | 0.4392 | 0.7284   |
+| 0.3812        | 4.43  | 5500  | 1.3308          | 0.4205    | 0.5146 | 0.4628 | 0.7351   |
+| 0.421         | 4.83  | 6000  | 1.3031          | 0.4275    | 0.5225 | 0.4702 | 0.7386   |
+| 0.3157        | 5.24  | 6500  | 1.3943          | 0.4132    | 0.5040 | 0.4541 | 0.7293   |
+| 0.3072        | 5.64  | 7000  | 1.4087          | 0.4303    | 0.5185 | 0.4703 | 0.7396   |
+| 0.3436        | 6.04  | 7500  | 1.4197          | 0.4461    | 0.5251 | 0.4824 | 0.7363   |
+| 0.2774        | 6.45  | 8000  | 1.4249          | 0.4275    | 0.5225 | 0.4702 | 0.7377   |
+| 0.2629        | 6.85  | 8500  | 1.4811          | 0.4580    | 0.5344 | 0.4933 | 0.7327   |
+| 0.2271        | 7.25  | 9000  | 1.5576          | 0.4733    | 0.5397 | 0.5043 | 0.7415   |
+| 0.235         | 7.66  | 9500  | 1.5468          | 0.4730    | 0.5569 | 0.5115 | 0.7401   |
+| 0.2415        | 8.06  | 10000 | 1.5956          | 0.4730    | 0.5437 | 0.5058 | 0.7433   |
+| 0.1826        | 8.46  | 10500 | 1.6168          | 0.4455    | 0.5410 | 0.4886 | 0.7413   |
+| 0.2083        | 8.86  | 11000 | 1.5866          | 0.4505    | 0.5423 | 0.4922 | 0.7413   |
+| 0.2169        | 9.27  | 11500 | 1.5974          | 0.4708    | 0.5437 | 0.5046 | 0.7468   |
+| 0.1747        | 9.67  | 12000 | 1.6219          | 0.4567    | 0.5437 | 0.4964 | 0.7405   |
+
+
+### Framework versions
+
+- Transformers 4.36.0.dev0
+- Pytorch 2.1.0+cu118
+- Datasets 2.15.0
+- Tokenizers 0.15.0

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

all_results.json

@@ -0,0 +1,25 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.7401096282173498,
+    "eval_f1": 0.511543134872418,
+    "eval_loss": 1.5468007326126099,
+    "eval_precision": 0.4730337078651685,
+    "eval_recall": 0.5568783068783069,
+    "eval_runtime": 3.8992,
+    "eval_samples": 304,
+    "eval_samples_per_second": 77.965,
+    "eval_steps_per_second": 38.983,
+    "predict_accuracy": 0.800990916597853,
+    "predict_f1": 0.5826558265582655,
+    "predict_loss": 1.046440839767456,
+    "predict_precision": 0.5402010050251256,
+    "predict_recall": 0.6323529411764706,
+    "predict_runtime": 1.6809,
+    "predict_samples_per_second": 77.935,
+    "predict_steps_per_second": 39.265,
+    "train_loss": 0.5249143927833133,
+    "train_runtime": 2240.4976,
+    "train_samples": 2481,
+    "train_samples_per_second": 11.073,
+    "train_steps_per_second": 5.539
+}

config.json

@@ -0,0 +1,122 @@
+{
+  "_name_or_path": "microsoft/deberta-v3-base",
+  "architectures": [
+    "DebertaV2ForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": "ner",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "B-ATTACK_MOTIVATION",
+    "1": "B-ATTACK_PATTERN",
+    "2": "B-ATTACK_RESOURCE_LEVEL",
+    "3": "B-ATTACK_SOPHISTICATION_LEVEL",
+    "4": "B-ATTACK_TOOL",
+    "5": "B-CAMPAIGN",
+    "6": "B-COURSE_OF_ACTION",
+    "7": "B-GENERAL_IDENTITY",
+    "8": "B-GENERAL_TOOL",
+    "9": "B-IMPACT",
+    "10": "B-INDICATOR",
+    "11": "B-INFRASTRUCTURE",
+    "12": "B-INTRUSION_SET",
+    "13": "B-LOCATION",
+    "14": "B-MALWARE",
+    "15": "B-MALWARE_ANALYSIS",
+    "16": "B-OBSERVED_DATA",
+    "17": "B-THREAT_ACTOR",
+    "18": "B-VICTIM_IDENTITY",
+    "19": "B-VULNERABILITY",
+    "20": "I-ATTACK_MOTIVATION",
+    "21": "I-ATTACK_PATTERN",
+    "22": "I-ATTACK_RESOURCE_LEVEL",
+    "23": "I-ATTACK_SOPHISTICATION_LEVEL",
+    "24": "I-ATTACK_TOOL",
+    "25": "I-CAMPAIGN",
+    "26": "I-COURSE_OF_ACTION",
+    "27": "I-GENERAL_IDENTITY",
+    "28": "I-GENERAL_TOOL",
+    "29": "I-IMPACT",
+    "30": "I-INDICATOR",
+    "31": "I-INFRASTRUCTURE",
+    "32": "I-INTRUSION_SET",
+    "33": "I-LOCATION",
+    "34": "I-MALWARE",
+    "35": "I-MALWARE_ANALYSIS",
+    "36": "I-OBSERVED_DATA",
+    "37": "I-THREAT_ACTOR",
+    "38": "I-VICTIM_IDENTITY",
+    "39": "I-VULNERABILITY",
+    "40": "O"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "B-ATTACK_MOTIVATION": 0,
+    "B-ATTACK_PATTERN": 1,
+    "B-ATTACK_RESOURCE_LEVEL": 2,
+    "B-ATTACK_SOPHISTICATION_LEVEL": 3,
+    "B-ATTACK_TOOL": 4,
+    "B-CAMPAIGN": 5,
+    "B-COURSE_OF_ACTION": 6,
+    "B-GENERAL_IDENTITY": 7,
+    "B-GENERAL_TOOL": 8,
+    "B-IMPACT": 9,
+    "B-INDICATOR": 10,
+    "B-INFRASTRUCTURE": 11,
+    "B-INTRUSION_SET": 12,
+    "B-LOCATION": 13,
+    "B-MALWARE": 14,
+    "B-MALWARE_ANALYSIS": 15,
+    "B-OBSERVED_DATA": 16,
+    "B-THREAT_ACTOR": 17,
+    "B-VICTIM_IDENTITY": 18,
+    "B-VULNERABILITY": 19,
+    "I-ATTACK_MOTIVATION": 20,
+    "I-ATTACK_PATTERN": 21,
+    "I-ATTACK_RESOURCE_LEVEL": 22,
+    "I-ATTACK_SOPHISTICATION_LEVEL": 23,
+    "I-ATTACK_TOOL": 24,
+    "I-CAMPAIGN": 25,
+    "I-COURSE_OF_ACTION": 26,
+    "I-GENERAL_IDENTITY": 27,
+    "I-GENERAL_TOOL": 28,
+    "I-IMPACT": 29,
+    "I-INDICATOR": 30,
+    "I-INFRASTRUCTURE": 31,
+    "I-INTRUSION_SET": 32,
+    "I-LOCATION": 33,
+    "I-MALWARE": 34,
+    "I-MALWARE_ANALYSIS": 35,
+    "I-OBSERVED_DATA": 36,
+    "I-THREAT_ACTOR": 37,
+    "I-VICTIM_IDENTITY": 38,
+    "I-VULNERABILITY": 39,
+    "O": 40
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.0.dev0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

eval_results.json

@@ -0,0 +1,12 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.7401096282173498,
+    "eval_f1": 0.511543134872418,
+    "eval_loss": 1.5468007326126099,
+    "eval_precision": 0.4730337078651685,
+    "eval_recall": 0.5568783068783069,
+    "eval_runtime": 3.8992,
+    "eval_samples": 304,
+    "eval_samples_per_second": 77.965,
+    "eval_steps_per_second": 38.983
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:806fcca27eeebf5670c37b296b6ccca058ae65140514f97a5670358de6d802ca
+size 735476708

predict_results.json

@@ -0,0 +1,10 @@
+{
+    "predict_accuracy": 0.800990916597853,
+    "predict_f1": 0.5826558265582655,
+    "predict_loss": 1.046440839767456,
+    "predict_precision": 0.5402010050251256,
+    "predict_recall": 0.6323529411764706,
+    "predict_runtime": 1.6809,
+    "predict_samples_per_second": 77.935,
+    "predict_steps_per_second": 39.265
+}

predictions.txt

@@ -0,0 +1,131 @@
+O O O B-ATTACK_TOOL O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O O O B-GENERAL_IDENTITY O O B-MALWARE O O O O O O O O O I-ATTACK_TOOL O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O B-CAMPAIGN O O O O O O O O O O O O O O O O O O
+O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O O O O O B-LOCATION O B-LOCATION O O O O O O O O O O O O O
+O O O O O O O O O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O O O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O O I-MALWARE_ANALYSIS O O O I-MALWARE_ANALYSIS O O O O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O O B-ATTACK_MOTIVATION O B-THREAT_ACTOR I-THREAT_ACTOR O O O B-THREAT_ACTOR I-THREAT_ACTOR O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O B-VICTIM_IDENTITY O O O O O B-MALWARE O
+O O O B-MALWARE O O O O O O O O O O O O O O B-THREAT_ACTOR O O O O O O O O O O
+O B-MALWARE B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O O O O O O B-INTRUSION_SET B-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET O
+B-MALWARE O O O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O B-CAMPAIGN O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O
+O O O O O O O B-THREAT_ACTOR O O O B-THREAT_ACTOR O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O B-VICTIM_IDENTITY O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O
+O O O B-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O B-MALWARE I-MALWARE O O O B-VULNERABILITY I-VULNERABILITY O B-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+O O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O B-IMPACT I-IMPACT I-IMPACT B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O O B-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O O B-GENERAL_TOOL I-GENERAL_TOOL O
+O O O O O O O O O O O O B-CAMPAIGN O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O B-THREAT_ACTOR O O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O B-ATTACK_TOOL I-ATTACK_TOOL O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-INDICATOR O O O B-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET O
+O O O O O O O O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O O O O O
+O O O O O O O B-VICTIM_IDENTITY O O O O O O O O O O O O O O O O O O O O O O
+O O B-MALWARE O B-THREAT_ACTOR O O B-ATTACK_TOOL I-ATTACK_TOOL I-ATTACK_TOOL O O O O O O
+O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O B-THREAT_ACTOR I-THREAT_ACTOR O
+O O O O I-INTRUSION_SET O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O B-THREAT_ACTOR O O B-ATTACK_PATTERN O O O O O O O O O O O O O O O O O O B-VICTIM_IDENTITY O
+B-THREAT_ACTOR O O O O O O O O O O O O O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O B-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O
+B-MALWARE O O O O O O O B-MALWARE I-MALWARE I-MALWARE O O O O O O O B-GENERAL_TOOL O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O O O O O
+O O O O O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+O O O O O O O O O O O O O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O
+O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O B-VICTIM_IDENTITY O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-THREAT_ACTOR I-THREAT_ACTOR O
+O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-GENERAL_IDENTITY O O O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O I-INFRASTRUCTURE O O O O O
+O O O O O O O I-ATTACK_PATTERN I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-GENERAL_IDENTITY O O O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O I-INFRASTRUCTURE O O O O O
+O O O O O O O O B-THREAT_ACTOR O O O O O O O O O O O O O O
+O O O O O O O O O O O O O O B-THREAT_ACTOR O O O O O O O B-THREAT_ACTOR O
+O O B-THREAT_ACTOR O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+O O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS
+O O O O B-CAMPAIGN I-CAMPAIGN O O O O O B-INTRUSION_SET I-INTRUSION_SET O B-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION B-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET O
+O O B-MALWARE O O O B-THREAT_ACTOR O B-ATTACK_TOOL I-ATTACK_TOOL I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O B-CAMPAIGN O
+B-THREAT_ACTOR I-THREAT_ACTOR O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O B-MALWARE O O B-INFRASTRUCTURE O I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O I-INFRASTRUCTURE O
+O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O O O O I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O
+B-THREAT_ACTOR O B-MALWARE O O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE O B-ATTACK_TOOL O
+B-THREAT_ACTOR B-ATTACK_PATTERN B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-ATTACK_PATTERN B-INFRASTRUCTURE O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-INDICATOR I-INDICATOR O O O O B-INFRASTRUCTURE O O
+B-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O B-INFRASTRUCTURE O O O O
+O B-GENERAL_IDENTITY O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-VULNERABILITY O O O O
+O O O O B-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O B-CAMPAIGN I-CAMPAIGN O O O O O O O O O O O
+O B-THREAT_ACTOR O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O B-ATTACK_TOOL I-ATTACK_TOOL O
+O O O O O B-THREAT_ACTOR O B-MALWARE O O O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-INFRASTRUCTURE I-MALWARE_ANALYSIS O O O O O O B-ATTACK_TOOL O I-MALWARE_ANALYSIS O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O O O O O B-LOCATION O B-LOCATION O O O O O O O O O O O O O
+B-MALWARE I-MALWARE O O O O O I-OBSERVED_DATA O O O O O O O O O O O O O O O
+O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O B-ATTACK_TOOL O O O I-MALWARE_ANALYSIS O O O O I-MALWARE_ANALYSIS O O O O O O O O O O O O O O O
+O O O B-INTRUSION_SET I-CAMPAIGN O B-VICTIM_IDENTITY I-VICTIM_IDENTITY
+O O O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+B-MALWARE O O O O O O O O O O O O B-ATTACK_TOOL O B-ATTACK_TOOL O O O O O O O O
+O O O O O B-MALWARE I-MALWARE O
+O O O O O O B-THREAT_ACTOR O O O O O O O O O O O O O O O O O O O O O O
+B-COURSE_OF_ACTION I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-GENERAL_IDENTITY O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O O O O O O
+O O O O I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O B-MALWARE I-MALWARE I-MALWARE I-MALWARE I-MALWARE O O B-MALWARE O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O O O O O O O O O O O O O O O O O I-MALWARE_ANALYSIS O O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O
+B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O B-OBSERVED_DATA O O B-MALWARE O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA
+B-ATTACK_TOOL O O O O O O O O O O O O O O O
+O O O O O B-GENERAL_IDENTITY O O O O B-LOCATION O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-GENERAL_IDENTITY O O O O
+B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA B-INFRASTRUCTURE I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O O B-MALWARE O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-OBSERVED_DATA O O I-OBSERVED_DATA I-OBSERVED_DATA B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O I-MALWARE_ANALYSIS O I-ATTACK_PATTERN O O O O O O O O
+O O O O O O O B-THREAT_ACTOR O O O
+O O O O O O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-GENERAL_IDENTITY O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O
+O O O O O O O O O O O O O O O O O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O O B-MALWARE O O O O O O O O O O
+B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O O O O B-THREAT_ACTOR O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O B-THREAT_ACTOR O O O B-THREAT_ACTOR O O O O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O B-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O
+O O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O I-COURSE_OF_ACTION O B-COURSE_OF_ACTION O I-COURSE_OF_ACTION O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-THREAT_ACTOR O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN I-ATTACK_PATTERN O O B-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O
+O O O O O O B-ATTACK_TOOL I-ATTACK_TOOL O O O O
+O O O B-VULNERABILITY O O O O O O O O O B-ATTACK_PATTERN O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O B-THREAT_ACTOR O O O O O
+O O O O O O B-THREAT_ACTOR O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-LOCATION I-LOCATION O O O O B-LOCATION O O O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O O O O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O
+O O O O O O O O O O O O O B-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN O O O O O O
+O O I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O O I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O B-LOCATION I-LOCATION O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O O O B-GENERAL_TOOL I-GENERAL_TOOL O
+O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-THREAT_ACTOR O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O B-THREAT_ACTOR O
+O O O O O O B-COURSE_OF_ACTION O O O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-ATTACK_TOOL O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O
+O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O B-GENERAL_IDENTITY O O B-VULNERABILITY I-VULNERABILITY O O O O O O O O O O O O O O O O
+O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O B-GENERAL_IDENTITY O O I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O O O O O O O O O B-GENERAL_TOOL O O O B-GENERAL_TOOL I-IMPACT
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE O B-INFRASTRUCTURE O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O I-COURSE_OF_ACTION O O B-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-GENERAL_TOOL O
+O O O O B-MALWARE I-MALWARE O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+O B-GENERAL_TOOL O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O B-MALWARE I-MALWARE O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+O O B-COURSE_OF_ACTION B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O O B-THREAT_ACTOR O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-LOCATION I-LOCATION O O O O B-LOCATION O O O
+B-OBSERVED_DATA I-ATTACK_PATTERN O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O O O O O O O O O O O O O O O O O B-MALWARE O O O B-MALWARE O B-MALWARE O O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O
+O O O B-VULNERABILITY O O O O O O O O O B-ATTACK_PATTERN O
+O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+O O O O O O O B-THREAT_ACTOR O O O O O O O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O B-ATTACK_MOTIVATION O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O B-ATTACK_MOTIVATION O
+B-THREAT_ACTOR O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR O
+O B-GENERAL_IDENTITY O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O
+O O O O O O O O O B-MALWARE O O O O B-MALWARE I-CAMPAIGN O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O B-ATTACK_TOOL I-ATTACK_TOOL I-ATTACK_TOOL O O O O O O O O O O O
+O O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O B-VULNERABILITY O O O O O O O O O O O O O
+O O O O B-MALWARE O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O

runs/Dec01_11-43-34_7b631c1718a6/events.out.tfevents.1701431028.7b631c1718a6.6113.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ce7249b9bb7f846906027d7bae0d9358af69a17f94b57e42868849ed159f12
+size 22254

runs/Dec01_11-43-34_7b631c1718a6/events.out.tfevents.1701433280.7b631c1718a6.6113.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74307ad7715b5fefa2b986ff1cc079f2cee7d1f7796ab94653e3b02635fcf26c
+size 560

runs/Nov21_17-29-54_1fcd30357cc4/events.out.tfevents.1700587809.1fcd30357cc4.11686.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:464dd4ee74ae80be98477450c95bdb636b3f6774fe1f34ab0f9b19086e2e094d
+size 34177

runs/Nov21_17-29-54_1fcd30357cc4/events.out.tfevents.1700589991.1fcd30357cc4.11686.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2579d1698421074e2391a0d00b068098e05fbb5ce91079668526ecc8e4ca1ade
+size 560

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 10.0,
+    "train_loss": 0.5249143927833133,
+    "train_runtime": 2240.4976,
+    "train_samples": 2481,
+    "train_samples_per_second": 11.073,
+    "train_steps_per_second": 5.539
+}

trainer_state.json

@@ -0,0 +1,461 @@
+{
+  "best_metric": 0.511543134872418,
+  "best_model_checkpoint": "/content/our_data/checkpoint-9500",
+  "epoch": 10.0,
+  "eval_steps": 500,
+  "global_step": 12410,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.91941982272361e-05,
+      "loss": 1.7886,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_accuracy": 0.6168970448045757,
+      "eval_f1": 0.19641754169240272,
+      "eval_loss": 1.5075236558914185,
+      "eval_precision": 0.1842410196987254,
+      "eval_recall": 0.21031746031746032,
+      "eval_runtime": 4.697,
+      "eval_samples_per_second": 64.723,
+      "eval_steps_per_second": 32.361,
+      "step": 500
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.83883964544722e-05,
+      "loss": 1.3644,
+      "step": 1000
+    },
+    {
+      "epoch": 0.81,
+      "eval_accuracy": 0.6491897044804575,
+      "eval_f1": 0.26658972879399884,
+      "eval_loss": 1.3342480659484863,
+      "eval_precision": 0.23643807574206754,
+      "eval_recall": 0.3055555555555556,
+      "eval_runtime": 3.8755,
+      "eval_samples_per_second": 78.442,
+      "eval_steps_per_second": 39.221,
+      "step": 1000
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 1.75825946817083e-05,
+      "loss": 1.1181,
+      "step": 1500
+    },
+    {
+      "epoch": 1.21,
+      "eval_accuracy": 0.6812440419447092,
+      "eval_f1": 0.3241626794258373,
+      "eval_loss": 1.2655349969863892,
+      "eval_precision": 0.29585152838427947,
+      "eval_recall": 0.3584656084656085,
+      "eval_runtime": 4.9077,
+      "eval_samples_per_second": 61.944,
+      "eval_steps_per_second": 30.972,
+      "step": 1500
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 1.67767929089444e-05,
+      "loss": 0.9833,
+      "step": 2000
+    },
+    {
+      "epoch": 1.61,
+      "eval_accuracy": 0.6777883698760725,
+      "eval_f1": 0.33541785105173394,
+      "eval_loss": 1.2368232011795044,
+      "eval_precision": 0.29411764705882354,
+      "eval_recall": 0.39021164021164023,
+      "eval_runtime": 4.7308,
+      "eval_samples_per_second": 64.26,
+      "eval_steps_per_second": 32.13,
+      "step": 2000
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 1.59709911361805e-05,
+      "loss": 0.9036,
+      "step": 2500
+    },
+    {
+      "epoch": 2.01,
+      "eval_accuracy": 0.7023355576739753,
+      "eval_f1": 0.37717121588089325,
+      "eval_loss": 1.268211007118225,
+      "eval_precision": 0.35514018691588783,
+      "eval_recall": 0.4021164021164021,
+      "eval_runtime": 3.7593,
+      "eval_samples_per_second": 80.867,
+      "eval_steps_per_second": 40.434,
+      "step": 2500
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.5165189363416601e-05,
+      "loss": 0.7102,
+      "step": 3000
+    },
+    {
+      "epoch": 2.42,
+      "eval_accuracy": 0.7159199237368923,
+      "eval_f1": 0.40775558166862513,
+      "eval_loss": 1.2175697088241577,
+      "eval_precision": 0.3668076109936575,
+      "eval_recall": 0.458994708994709,
+      "eval_runtime": 4.1645,
+      "eval_samples_per_second": 72.998,
+      "eval_steps_per_second": 36.499,
+      "step": 3000
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 1.4359387590652701e-05,
+      "loss": 0.6868,
+      "step": 3500
+    },
+    {
+      "epoch": 2.82,
+      "eval_accuracy": 0.7147283126787417,
+      "eval_f1": 0.41918294849023086,
+      "eval_loss": 1.2170379161834717,
+      "eval_precision": 0.37942122186495175,
+      "eval_recall": 0.46825396825396826,
+      "eval_runtime": 4.987,
+      "eval_samples_per_second": 60.958,
+      "eval_steps_per_second": 30.479,
+      "step": 3500
+    },
+    {
+      "epoch": 3.22,
+      "learning_rate": 1.35535858178888e-05,
+      "loss": 0.5671,
+      "step": 4000
+    },
+    {
+      "epoch": 3.22,
+      "eval_accuracy": 0.7259294566253575,
+      "eval_f1": 0.4366863905325444,
+      "eval_loss": 1.2602918148040771,
+      "eval_precision": 0.3950749464668094,
+      "eval_recall": 0.4880952380952381,
+      "eval_runtime": 3.9634,
+      "eval_samples_per_second": 76.702,
+      "eval_steps_per_second": 38.351,
+      "step": 4000
+    },
+    {
+      "epoch": 3.63,
+      "learning_rate": 1.27477840451249e-05,
+      "loss": 0.4878,
+      "step": 4500
+    },
+    {
+      "epoch": 3.63,
+      "eval_accuracy": 0.7333174451858914,
+      "eval_f1": 0.4432930339666091,
+      "eval_loss": 1.2460124492645264,
+      "eval_precision": 0.3924566768603466,
+      "eval_recall": 0.5092592592592593,
+      "eval_runtime": 5.1761,
+      "eval_samples_per_second": 58.732,
+      "eval_steps_per_second": 29.366,
+      "step": 4500
+    },
+    {
+      "epoch": 4.03,
+      "learning_rate": 1.1941982272361e-05,
+      "loss": 0.4942,
+      "step": 5000
+    },
+    {
+      "epoch": 4.03,
+      "eval_accuracy": 0.7284318398474738,
+      "eval_f1": 0.43920145190562615,
+      "eval_loss": 1.314683198928833,
+      "eval_precision": 0.40468227424749165,
+      "eval_recall": 0.4801587301587302,
+      "eval_runtime": 3.8399,
+      "eval_samples_per_second": 79.169,
+      "eval_steps_per_second": 39.585,
+      "step": 5000
+    },
+    {
+      "epoch": 4.43,
+      "learning_rate": 1.11361804995971e-05,
+      "loss": 0.3812,
+      "step": 5500
+    },
+    {
+      "epoch": 4.43,
+      "eval_accuracy": 0.7351048617731173,
+      "eval_f1": 0.46281975014872095,
+      "eval_loss": 1.3308484554290771,
+      "eval_precision": 0.4205405405405405,
+      "eval_recall": 0.5145502645502645,
+      "eval_runtime": 4.4524,
+      "eval_samples_per_second": 68.278,
+      "eval_steps_per_second": 34.139,
+      "step": 5500
+    },
+    {
+      "epoch": 4.83,
+      "learning_rate": 1.0330378726833199e-05,
+      "loss": 0.421,
+      "step": 6000
+    },
+    {
+      "epoch": 4.83,
+      "eval_accuracy": 0.738560533841754,
+      "eval_f1": 0.4702380952380952,
+      "eval_loss": 1.3030654191970825,
+      "eval_precision": 0.4274891774891775,
+      "eval_recall": 0.5224867724867724,
+      "eval_runtime": 4.5652,
+      "eval_samples_per_second": 66.59,
+      "eval_steps_per_second": 33.295,
+      "step": 6000
+    },
+    {
+      "epoch": 5.24,
+      "learning_rate": 9.5245769540693e-06,
+      "loss": 0.3157,
+      "step": 6500
+    },
+    {
+      "epoch": 5.24,
+      "eval_accuracy": 0.7292659675881792,
+      "eval_f1": 0.4541120381406436,
+      "eval_loss": 1.394273042678833,
+      "eval_precision": 0.41323210412147504,
+      "eval_recall": 0.503968253968254,
+      "eval_runtime": 4.8385,
+      "eval_samples_per_second": 62.83,
+      "eval_steps_per_second": 31.415,
+      "step": 6500
+    },
+    {
+      "epoch": 5.64,
+      "learning_rate": 8.7187751813054e-06,
+      "loss": 0.3072,
+      "step": 7000
+    },
+    {
+      "epoch": 5.64,
+      "eval_accuracy": 0.7396329837940896,
+      "eval_f1": 0.47030593881223753,
+      "eval_loss": 1.4086649417877197,
+      "eval_precision": 0.43029637760702527,
+      "eval_recall": 0.5185185185185185,
+      "eval_runtime": 3.6352,
+      "eval_samples_per_second": 83.626,
+      "eval_steps_per_second": 41.813,
+      "step": 7000
+    },
+    {
+      "epoch": 6.04,
+      "learning_rate": 7.9129734085415e-06,
+      "loss": 0.3436,
+      "step": 7500
+    },
+    {
+      "epoch": 6.04,
+      "eval_accuracy": 0.7362964728312679,
+      "eval_f1": 0.48238153098420417,
+      "eval_loss": 1.4196876287460327,
+      "eval_precision": 0.4460674157303371,
+      "eval_recall": 0.5251322751322751,
+      "eval_runtime": 3.9627,
+      "eval_samples_per_second": 76.716,
+      "eval_steps_per_second": 38.358,
+      "step": 7500
+    },
+    {
+      "epoch": 6.45,
+      "learning_rate": 7.107171635777599e-06,
+      "loss": 0.2774,
+      "step": 8000
+    },
+    {
+      "epoch": 6.45,
+      "eval_accuracy": 0.7377264061010487,
+      "eval_f1": 0.4702380952380952,
+      "eval_loss": 1.4248775243759155,
+      "eval_precision": 0.4274891774891775,
+      "eval_recall": 0.5224867724867724,
+      "eval_runtime": 4.1017,
+      "eval_samples_per_second": 74.116,
+      "eval_steps_per_second": 37.058,
+      "step": 8000
+    },
+    {
+      "epoch": 6.85,
+      "learning_rate": 6.301369863013699e-06,
+      "loss": 0.2629,
+      "step": 8500
+    },
+    {
+      "epoch": 6.85,
+      "eval_accuracy": 0.732721639656816,
+      "eval_f1": 0.4932844932844933,
+      "eval_loss": 1.4811160564422607,
+      "eval_precision": 0.4580498866213152,
+      "eval_recall": 0.5343915343915344,
+      "eval_runtime": 3.7972,
+      "eval_samples_per_second": 80.06,
+      "eval_steps_per_second": 40.03,
+      "step": 8500
+    },
+    {
+      "epoch": 7.25,
+      "learning_rate": 5.495568090249799e-06,
+      "loss": 0.2271,
+      "step": 9000
+    },
+    {
+      "epoch": 7.25,
+      "eval_accuracy": 0.7415395614871306,
+      "eval_f1": 0.5043263288009888,
+      "eval_loss": 1.5576499700546265,
+      "eval_precision": 0.4733178654292343,
+      "eval_recall": 0.5396825396825397,
+      "eval_runtime": 3.7692,
+      "eval_samples_per_second": 80.653,
+      "eval_steps_per_second": 40.326,
+      "step": 9000
+    },
+    {
+      "epoch": 7.66,
+      "learning_rate": 4.689766317485899e-06,
+      "loss": 0.235,
+      "step": 9500
+    },
+    {
+      "epoch": 7.66,
+      "eval_accuracy": 0.7401096282173498,
+      "eval_f1": 0.511543134872418,
+      "eval_loss": 1.5468007326126099,
+      "eval_precision": 0.4730337078651685,
+      "eval_recall": 0.5568783068783069,
+      "eval_runtime": 4.6282,
+      "eval_samples_per_second": 65.684,
+      "eval_steps_per_second": 32.842,
+      "step": 9500
+    },
+    {
+      "epoch": 8.06,
+      "learning_rate": 3.883964544721999e-06,
+      "loss": 0.2415,
+      "step": 10000
+    },
+    {
+      "epoch": 8.06,
+      "eval_accuracy": 0.7433269780743565,
+      "eval_f1": 0.5058461538461538,
+      "eval_loss": 1.5955547094345093,
+      "eval_precision": 0.47295742232451093,
+      "eval_recall": 0.5436507936507936,
+      "eval_runtime": 4.7141,
+      "eval_samples_per_second": 64.487,
+      "eval_steps_per_second": 32.244,
+      "step": 10000
+    },
+    {
+      "epoch": 8.46,
+      "learning_rate": 3.0781627719580986e-06,
+      "loss": 0.1826,
+      "step": 10500
+    },
+    {
+      "epoch": 8.46,
+      "eval_accuracy": 0.7413012392755005,
+      "eval_f1": 0.48864994026284353,
+      "eval_loss": 1.616766333580017,
+      "eval_precision": 0.4455337690631808,
+      "eval_recall": 0.541005291005291,
+      "eval_runtime": 4.6222,
+      "eval_samples_per_second": 65.77,
+      "eval_steps_per_second": 32.885,
+      "step": 10500
+    },
+    {
+      "epoch": 8.86,
+      "learning_rate": 2.2723609991941985e-06,
+      "loss": 0.2083,
+      "step": 11000
+    },
+    {
+      "epoch": 8.86,
+      "eval_accuracy": 0.7413012392755005,
+      "eval_f1": 0.49219687875150064,
+      "eval_loss": 1.5865833759307861,
+      "eval_precision": 0.45054945054945056,
+      "eval_recall": 0.5423280423280423,
+      "eval_runtime": 4.7524,
+      "eval_samples_per_second": 63.967,
+      "eval_steps_per_second": 31.984,
+      "step": 11000
+    },
+    {
+      "epoch": 9.27,
+      "learning_rate": 1.4665592264302982e-06,
+      "loss": 0.2169,
+      "step": 11500
+    },
+    {
+      "epoch": 9.27,
+      "eval_accuracy": 0.7467826501429933,
+      "eval_f1": 0.5046040515653776,
+      "eval_loss": 1.5974096059799194,
+      "eval_precision": 0.47079037800687284,
+      "eval_recall": 0.5436507936507936,
+      "eval_runtime": 3.7982,
+      "eval_samples_per_second": 80.038,
+      "eval_steps_per_second": 40.019,
+      "step": 11500
+    },
+    {
+      "epoch": 9.67,
+      "learning_rate": 6.607574536663981e-07,
+      "loss": 0.1747,
+      "step": 12000
+    },
+    {
+      "epoch": 9.67,
+      "eval_accuracy": 0.740467111534795,
+      "eval_f1": 0.4963768115942029,
+      "eval_loss": 1.6219406127929688,
+      "eval_precision": 0.45666666666666667,
+      "eval_recall": 0.5436507936507936,
+      "eval_runtime": 4.7158,
+      "eval_samples_per_second": 64.464,
+      "eval_steps_per_second": 32.232,
+      "step": 12000
+    },
+    {
+      "epoch": 10.0,
+      "step": 12410,
+      "total_flos": 499833589185600.0,
+      "train_loss": 0.5249143927833133,
+      "train_runtime": 2240.4976,
+      "train_samples_per_second": 11.073,
+      "train_steps_per_second": 5.539
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 12410,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "total_flos": 499833589185600.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1d339f8d0f6588b82a985da75159742e86ce8b6df16cf80bab7d14db1a5c9c5
+size 4664