Initial Commit

README.md

@@ -0,0 +1,71 @@
+---
+license: mit
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: best_model
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# best_model
+
+This model is a fine-tuned version of [microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext](https://huggingface.co/microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2833
+- Accuracy: 0.8942
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 64
+- eval_batch_size: 64
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 2.0
+
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss | Accuracy |
+|:-------------:|:-----:|:-----:|:---------------:|:--------:|
+| 0.3604        | 0.14  | 5000  | 0.3162          | 0.8821   |
+| 0.3326        | 0.29  | 10000 | 0.3112          | 0.8843   |
+| 0.3293        | 0.43  | 15000 | 0.3044          | 0.8870   |
+| 0.3246        | 0.58  | 20000 | 0.3040          | 0.8871   |
+| 0.32          | 0.72  | 25000 | 0.2969          | 0.8888   |
+| 0.3143        | 0.87  | 30000 | 0.2929          | 0.8903   |
+| 0.3095        | 1.01  | 35000 | 0.2917          | 0.8899   |
+| 0.2844        | 1.16  | 40000 | 0.2957          | 0.8886   |
+| 0.2778        | 1.3   | 45000 | 0.2943          | 0.8906   |
+| 0.2779        | 1.45  | 50000 | 0.2890          | 0.8935   |
+| 0.2752        | 1.59  | 55000 | 0.2881          | 0.8919   |
+| 0.2736        | 1.74  | 60000 | 0.2835          | 0.8944   |
+| 0.2725        | 1.88  | 65000 | 0.2833          | 0.8942   |
+
+
+### Framework versions
+
+- Transformers 4.18.0.dev0
+- Pytorch 1.10.0+cu111
+- Datasets 1.18.3
+- Tokenizers 0.11.6

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 2.0,
+    "eval_accuracy": 0.8941656351089478,
+    "eval_loss": 0.28330111503601074,
+    "eval_runtime": 142.8376,
+    "eval_samples": 28932,
+    "eval_samples_per_second": 202.552,
+    "eval_steps_per_second": 3.171,
+    "train_loss": 0.30218412260715205,
+    "train_runtime": 63905.8286,
+    "train_samples": 2211861,
+    "train_samples_per_second": 69.223,
+    "train_steps_per_second": 1.082
+}

config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "BACKGROUND",
+    "1": "CONCLUSIONS",
+    "2": "METHODS",
+    "3": "OBJECTIVE",
+    "4": "RESULTS"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "BACKGROUND": 0,
+    "CONCLUSIONS": 1,
+    "METHODS": 2,
+    "OBJECTIVE": 3,
+    "RESULTS": 4
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.18.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.0,
+    "eval_accuracy": 0.8941656351089478,
+    "eval_loss": 0.28330111503601074,
+    "eval_runtime": 142.8376,
+    "eval_samples": 28932,
+    "eval_samples_per_second": 202.552,
+    "eval_steps_per_second": 3.171
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7cfce708e03d354d73968f3d50dc459590f6e6bb57434735aacb33a23c04234
+size 438029357

runs/Mar03_20-40-16_41ce11bcd200/1646340340.8113117/events.out.tfevents.1646340340.41ce11bcd200.1075.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9cbaa92c7e952566d14e530a77f80e4f5f72d8b02392f431ac779616fec5ec7
+size 4772

runs/Mar03_20-40-16_41ce11bcd200/events.out.tfevents.1646340340.41ce11bcd200.1075.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca20cde9188e3a32f916c315ddc6ac9f1afc3c444f4e717f365201a2db8b3bf1
+size 10256

runs/Mar03_20-40-16_41ce11bcd200/events.out.tfevents.1646404390.41ce11bcd200.1075.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46f736b6637ae9d2d6be77eda2ed4b2ba7995738caf2d744489b614ffdd29e9d
+size 369

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "special_tokens_map_file": null, "name_or_path": "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext", "do_basic_tokenize": true, "never_split": null, "tokenizer_class": "BertTokenizer"}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.30218412260715205,
+    "train_runtime": 63905.8286,
+    "train_samples": 2211861,
+    "train_samples_per_second": 69.223,
+    "train_steps_per_second": 1.082
+}

trainer_state.json

@@ -0,0 +1,220 @@
+{
+  "best_metric": 0.28330111503601074,
+  "best_model_checkpoint": "/content/best_model/checkpoint-65000",
+  "epoch": 2.0,
+  "global_step": 69122,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.638320650444142e-05,
+      "loss": 0.3604,
+      "step": 5000
+    },
+    {
+      "epoch": 0.14,
+      "eval_accuracy": 0.8821028470993042,
+      "eval_loss": 0.3162487745285034,
+      "eval_runtime": 142.0782,
+      "eval_samples_per_second": 203.634,
+      "eval_steps_per_second": 3.188,
+      "step": 5000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.276641300888285e-05,
+      "loss": 0.3326,
+      "step": 10000
+    },
+    {
+      "epoch": 0.29,
+      "eval_accuracy": 0.8842803835868835,
+      "eval_loss": 0.311200350522995,
+      "eval_runtime": 142.1706,
+      "eval_samples_per_second": 203.502,
+      "eval_steps_per_second": 3.186,
+      "step": 10000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.914961951332427e-05,
+      "loss": 0.3293,
+      "step": 15000
+    },
+    {
+      "epoch": 0.43,
+      "eval_accuracy": 0.8869763612747192,
+      "eval_loss": 0.3043750524520874,
+      "eval_runtime": 142.2145,
+      "eval_samples_per_second": 203.439,
+      "eval_steps_per_second": 3.185,
+      "step": 15000
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 3.5532826017765694e-05,
+      "loss": 0.3246,
+      "step": 20000
+    },
+    {
+      "epoch": 0.58,
+      "eval_accuracy": 0.8871491551399231,
+      "eval_loss": 0.30401167273521423,
+      "eval_runtime": 142.2635,
+      "eval_samples_per_second": 203.369,
+      "eval_steps_per_second": 3.184,
+      "step": 20000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.191603252220711e-05,
+      "loss": 0.32,
+      "step": 25000
+    },
+    {
+      "epoch": 0.72,
+      "eval_accuracy": 0.8887736797332764,
+      "eval_loss": 0.29694026708602905,
+      "eval_runtime": 142.2955,
+      "eval_samples_per_second": 203.323,
+      "eval_steps_per_second": 3.184,
+      "step": 25000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.829923902664854e-05,
+      "loss": 0.3143,
+      "step": 30000
+    },
+    {
+      "epoch": 0.87,
+      "eval_accuracy": 0.8903290629386902,
+      "eval_loss": 0.2928813695907593,
+      "eval_runtime": 142.3659,
+      "eval_samples_per_second": 203.223,
+      "eval_steps_per_second": 3.182,
+      "step": 30000
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 2.468244553108996e-05,
+      "loss": 0.3095,
+      "step": 35000
+    },
+    {
+      "epoch": 1.01,
+      "eval_accuracy": 0.8899142742156982,
+      "eval_loss": 0.29173970222473145,
+      "eval_runtime": 142.5262,
+      "eval_samples_per_second": 202.994,
+      "eval_steps_per_second": 3.178,
+      "step": 35000
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 2.106565203553138e-05,
+      "loss": 0.2844,
+      "step": 40000
+    },
+    {
+      "epoch": 1.16,
+      "eval_accuracy": 0.8886008858680725,
+      "eval_loss": 0.29569417238235474,
+      "eval_runtime": 142.5524,
+      "eval_samples_per_second": 202.957,
+      "eval_steps_per_second": 3.178,
+      "step": 40000
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 1.7448858539972804e-05,
+      "loss": 0.2778,
+      "step": 45000
+    },
+    {
+      "epoch": 1.3,
+      "eval_accuracy": 0.890640139579773,
+      "eval_loss": 0.2942551076412201,
+      "eval_runtime": 142.6301,
+      "eval_samples_per_second": 202.846,
+      "eval_steps_per_second": 3.176,
+      "step": 45000
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 1.3832065044414225e-05,
+      "loss": 0.2779,
+      "step": 50000
+    },
+    {
+      "epoch": 1.45,
+      "eval_accuracy": 0.8934743404388428,
+      "eval_loss": 0.28896576166152954,
+      "eval_runtime": 142.7675,
+      "eval_samples_per_second": 202.651,
+      "eval_steps_per_second": 3.173,
+      "step": 50000
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 1.0215271548855646e-05,
+      "loss": 0.2752,
+      "step": 55000
+    },
+    {
+      "epoch": 1.59,
+      "eval_accuracy": 0.891884446144104,
+      "eval_loss": 0.28808724880218506,
+      "eval_runtime": 142.5778,
+      "eval_samples_per_second": 202.921,
+      "eval_steps_per_second": 3.177,
+      "step": 55000
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 6.598478053297069e-06,
+      "loss": 0.2736,
+      "step": 60000
+    },
+    {
+      "epoch": 1.74,
+      "eval_accuracy": 0.8943729996681213,
+      "eval_loss": 0.28354716300964355,
+      "eval_runtime": 142.8817,
+      "eval_samples_per_second": 202.489,
+      "eval_steps_per_second": 3.17,
+      "step": 60000
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 2.9816845577384916e-06,
+      "loss": 0.2725,
+      "step": 65000
+    },
+    {
+      "epoch": 1.88,
+      "eval_accuracy": 0.8941656351089478,
+      "eval_loss": 0.28330111503601074,
+      "eval_runtime": 142.8534,
+      "eval_samples_per_second": 202.529,
+      "eval_steps_per_second": 3.171,
+      "step": 65000
+    },
+    {
+      "epoch": 2.0,
+      "step": 69122,
+      "total_flos": 3.410043502198626e+17,
+      "train_loss": 0.30218412260715205,
+      "train_runtime": 63905.8286,
+      "train_samples_per_second": 69.223,
+      "train_steps_per_second": 1.082
+    }
+  ],
+  "max_steps": 69122,
+  "num_train_epochs": 2,
+  "total_flos": 3.410043502198626e+17,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:001fc1f97e17e76d08b64c17aa20b40aa8092fd7d6fe9632d3e543096a26c007
+size 2991