Initial commit from mrm8488

README.md

@@ -0,0 +1,78 @@
+---
+language:
+- en
+license: mit
+tags:
+- generated_from_trainer
+datasets:
+- glue
+metrics:
+- accuracy
+model-index:
+- name: deberta-v3-small
+  results:
+  - task:
+      name: Text Classification
+      type: text-classification
+    dataset:
+      name: GLUE QNLI
+      type: glue
+      args: qnli
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.9150649826102873
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# deberta-v3-small
+
+This model is a fine-tuned version of [microsoft/deberta-v3-small](https://huggingface.co/microsoft/deberta-v3-small) on the GLUE QNLI dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2143
+- Accuracy: 0.9151
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 5.0
+
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss | Accuracy |
+|:-------------:|:-----:|:-----:|:---------------:|:--------:|
+| 0.2823        | 1.0   | 6547  | 0.2143          | 0.9151   |
+| 0.1996        | 2.0   | 13094 | 0.2760          | 0.9103   |
+| 0.1327        | 3.0   | 19641 | 0.3293          | 0.9169   |
+| 0.0811        | 4.0   | 26188 | 0.4278          | 0.9193   |
+| 0.05          | 5.0   | 32735 | 0.5110          | 0.9176   |
+
+
+### Framework versions
+
+- Transformers 4.13.0.dev0
+- Pytorch 1.10.0+cu111
+- Datasets 1.16.1
+- Tokenizers 0.10.3

added_tokens.json

@@ -0,0 +1 @@
+{"[MASK]": 128000}

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 5.0,
+    "eval_accuracy": 0.9150649826102873,
+    "eval_loss": 0.2142563909292221,
+    "eval_runtime": 10.9674,
+    "eval_samples": 5463,
+    "eval_samples_per_second": 498.115,
+    "eval_steps_per_second": 62.276,
+    "train_loss": 0.15606709380213207,
+    "train_runtime": 1886.2061,
+    "train_samples": 104743,
+    "train_samples_per_second": 277.655,
+    "train_steps_per_second": 17.355
+}

config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "microsoft/deberta-v3-small",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": "qnli",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "entailment",
+    "1": "not_entailment"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "entailment": 0,
+    "not_entailment": 1
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.13.0.dev0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 5.0,
+    "eval_accuracy": 0.9150649826102873,
+    "eval_loss": 0.2142563909292221,
+    "eval_runtime": 10.9674,
+    "eval_samples": 5463,
+    "eval_samples_per_second": 498.115,
+    "eval_steps_per_second": 62.276
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aae4026d6f22c73fe93c21a9e0a065184d2c4e9c294ec4543ed78c17e4ac7ca
+size 567632875

runs/events.out.tfevents.1638818959.05f9258cf39b.715.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5d1956847edc1da2241195d32fb83a12bae4b8f355aef9259886fedaa9e0c1c
+size 10619

runs/events.out.tfevents.1638820857.05f9258cf39b.715.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4570f0b9d39477fc71aa679ad8c931f357f7649b7b10c60279c8333f1408954b
+size 369

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "split_by_punct": false, "sp_model_kwargs": {}, "vocab_type": "spm", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "microsoft/deberta-v3-small", "tokenizer_class": "DebertaV2Tokenizer"}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 5.0,
+    "train_loss": 0.15606709380213207,
+    "train_runtime": 1886.2061,
+    "train_samples": 104743,
+    "train_samples_per_second": 277.655,
+    "train_steps_per_second": 17.355
+}

trainer_state.json

@@ -0,0 +1,262 @@
+{
+  "best_metric": 0.2142563909292221,
+  "best_model_checkpoint": "/content/deberta-v3-small/checkpoint-6547",
+  "epoch": 5.0,
+  "global_step": 32735,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.15,
+      "learning_rate": 2.908354971742783e-05,
+      "loss": 0.3956,
+      "step": 1000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 2.816709943485566e-05,
+      "loss": 0.3285,
+      "step": 2000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.7250649152283487e-05,
+      "loss": 0.313,
+      "step": 3000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.633419886971132e-05,
+      "loss": 0.2956,
+      "step": 4000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.541774858713915e-05,
+      "loss": 0.2931,
+      "step": 5000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 2.450129830456698e-05,
+      "loss": 0.2823,
+      "step": 6000
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9150649826102873,
+      "eval_loss": 0.2142563909292221,
+      "eval_runtime": 10.8101,
+      "eval_samples_per_second": 505.359,
+      "eval_steps_per_second": 63.181,
+      "step": 6547
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 2.3584848021994806e-05,
+      "loss": 0.2467,
+      "step": 7000
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 2.2668397739422636e-05,
+      "loss": 0.1968,
+      "step": 8000
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 2.1751947456850465e-05,
+      "loss": 0.1994,
+      "step": 9000
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 2.08354971742783e-05,
+      "loss": 0.203,
+      "step": 10000
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 1.9919046891706125e-05,
+      "loss": 0.2003,
+      "step": 11000
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 1.9002596609133955e-05,
+      "loss": 0.1904,
+      "step": 12000
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 1.8086146326561784e-05,
+      "loss": 0.1996,
+      "step": 13000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9103056928427604,
+      "eval_loss": 0.27601319551467896,
+      "eval_runtime": 10.9101,
+      "eval_samples_per_second": 500.73,
+      "eval_steps_per_second": 62.603,
+      "step": 13094
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 1.7169696043989614e-05,
+      "loss": 0.1357,
+      "step": 14000
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 1.625324576141744e-05,
+      "loss": 0.1331,
+      "step": 15000
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 1.5336795478845274e-05,
+      "loss": 0.1317,
+      "step": 16000
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 1.4420345196273102e-05,
+      "loss": 0.1337,
+      "step": 17000
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 1.3503894913700933e-05,
+      "loss": 0.1313,
+      "step": 18000
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 1.2587444631128761e-05,
+      "loss": 0.1327,
+      "step": 19000
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9168954786747209,
+      "eval_loss": 0.3293353319168091,
+      "eval_runtime": 10.8188,
+      "eval_samples_per_second": 504.952,
+      "eval_steps_per_second": 63.131,
+      "step": 19641
+    },
+    {
+      "epoch": 3.05,
+      "learning_rate": 1.1670994348556591e-05,
+      "loss": 0.1108,
+      "step": 20000
+    },
+    {
+      "epoch": 3.21,
+      "learning_rate": 1.0754544065984421e-05,
+      "loss": 0.0817,
+      "step": 21000
+    },
+    {
+      "epoch": 3.36,
+      "learning_rate": 9.83809378341225e-06,
+      "loss": 0.0851,
+      "step": 22000
+    },
+    {
+      "epoch": 3.51,
+      "learning_rate": 8.921643500840079e-06,
+      "loss": 0.0787,
+      "step": 23000
+    },
+    {
+      "epoch": 3.67,
+      "learning_rate": 8.00519321826791e-06,
+      "loss": 0.0897,
+      "step": 24000
+    },
+    {
+      "epoch": 3.82,
+      "learning_rate": 7.088742935695739e-06,
+      "loss": 0.0917,
+      "step": 25000
+    },
+    {
+      "epoch": 3.97,
+      "learning_rate": 6.172292653123568e-06,
+      "loss": 0.0811,
+      "step": 26000
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.9192751235584844,
+      "eval_loss": 0.4277651309967041,
+      "eval_runtime": 10.6276,
+      "eval_samples_per_second": 514.037,
+      "eval_steps_per_second": 64.266,
+      "step": 26188
+    },
+    {
+      "epoch": 4.12,
+      "learning_rate": 5.255842370551398e-06,
+      "loss": 0.0578,
+      "step": 27000
+    },
+    {
+      "epoch": 4.28,
+      "learning_rate": 4.339392087979228e-06,
+      "loss": 0.0518,
+      "step": 28000
+    },
+    {
+      "epoch": 4.43,
+      "learning_rate": 3.4229418054070567e-06,
+      "loss": 0.052,
+      "step": 29000
+    },
+    {
+      "epoch": 4.58,
+      "learning_rate": 2.5064915228348865e-06,
+      "loss": 0.052,
+      "step": 30000
+    },
+    {
+      "epoch": 4.73,
+      "learning_rate": 1.5900412402627159e-06,
+      "loss": 0.0448,
+      "step": 31000
+    },
+    {
+      "epoch": 4.89,
+      "learning_rate": 6.735909576905453e-07,
+      "loss": 0.05,
+      "step": 32000
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.9176276771004942,
+      "eval_loss": 0.5109593868255615,
+      "eval_runtime": 10.8996,
+      "eval_samples_per_second": 501.212,
+      "eval_steps_per_second": 62.663,
+      "step": 32735
+    },
+    {
+      "epoch": 5.0,
+      "step": 32735,
+      "total_flos": 1.734440872113408e+16,
+      "train_loss": 0.15606709380213207,
+      "train_runtime": 1886.2061,
+      "train_samples_per_second": 277.655,
+      "train_steps_per_second": 17.355
+    }
+  ],
+  "max_steps": 32735,
+  "num_train_epochs": 5,
+  "total_flos": 1.734440872113408e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88a4cd8abdc7501e9f667c3605f9282b8e62a2fe1ba2844b1934d379173a587e
+size 2927