Bodo BERT Base with All Article of News and General Corpus

README.md

@@ -1,3 +1,55 @@
 ---
-license: cc-by-sa-4.0
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: brx-bert-base-mlm-scratch-article-v1
+  results: []
 ---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# brx-bert-base-mlm-scratch-article-v1
+
+This model was trained from scratch on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 8.4517
+- Accuracy: 0.0383
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.26.0.dev0
+- Pytorch 1.12.1+cu113
+- Datasets 2.8.0
+- Tokenizers 0.13.2

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.03827751196172249,
+    "eval_loss": 8.45165729522705,
+    "eval_runtime": 0.2649,
+    "eval_samples": 12,
+    "eval_samples_per_second": 45.295,
+    "eval_steps_per_second": 22.647,
+    "perplexity": 4682.827135132336,
+    "train_loss": 8.53661620345823,
+    "train_runtime": 1479.3105,
+    "train_samples": 5657,
+    "train_samples_per_second": 11.472,
+    "train_steps_per_second": 5.737
+}

config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "/home/sn/pretrained_models/huggingface_pretrained/brx-bert-base-mlm-scratch/pretrained-brx-bert-base-article",
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "mask_token": "[MASK]",
+  "max_len": 512,
+  "max_position_embeddings": 512,
+  "model_max_length": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token": "[PAD]",
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "sep_token": "[SEP]",
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.0.dev0",
+  "type_vocab_size": 2,
+  "unk_token": "[UNK]",
+  "use_cache": true,
+  "vocab_size": 50000
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.03827751196172249,
+    "eval_loss": 8.45165729522705,
+    "eval_runtime": 0.2649,
+    "eval_samples": 12,
+    "eval_samples_per_second": 45.295,
+    "eval_steps_per_second": 22.647,
+    "perplexity": 4682.827135132336
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:741f69b220783c70dd4290987c3baf1d59a1417f3588966bbd12baa13a377320
+size 498043179

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "name_or_path": "/home/sn/pretrained_models/huggingface_pretrained/brx-bert-base-mlm-scratch/pretrained-brx-bert-base-article",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 8.53661620345823,
+    "train_runtime": 1479.3105,
+    "train_samples": 5657,
+    "train_samples_per_second": 11.472,
+    "train_steps_per_second": 5.737
+}

trainer_state.json

@@ -0,0 +1,121 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "global_step": 8487,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.705431836927065e-05,
+      "loss": 8.8942,
+      "step": 500
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.41086367385413e-05,
+      "loss": 8.6759,
+      "step": 1000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.116295510781195e-05,
+      "loss": 8.6327,
+      "step": 1500
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 3.82172734770826e-05,
+      "loss": 8.6021,
+      "step": 2000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 3.5271591846353246e-05,
+      "loss": 8.5811,
+      "step": 2500
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 3.2325910215623894e-05,
+      "loss": 8.5453,
+      "step": 3000
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 2.938022858489455e-05,
+      "loss": 8.526,
+      "step": 3500
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 2.6434546954165195e-05,
+      "loss": 8.4978,
+      "step": 4000
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 2.3488865323435844e-05,
+      "loss": 8.4784,
+      "step": 4500
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 2.0543183692706496e-05,
+      "loss": 8.4831,
+      "step": 5000
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.759750206197714e-05,
+      "loss": 8.4657,
+      "step": 5500
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 1.4651820431247793e-05,
+      "loss": 8.464,
+      "step": 6000
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 1.170613880051844e-05,
+      "loss": 8.4781,
+      "step": 6500
+    },
+    {
+      "epoch": 2.47,
+      "learning_rate": 8.76045716978909e-06,
+      "loss": 8.4584,
+      "step": 7000
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 5.814775539059739e-06,
+      "loss": 8.436,
+      "step": 7500
+    },
+    {
+      "epoch": 2.83,
+      "learning_rate": 2.869093908330388e-06,
+      "loss": 8.4549,
+      "step": 8000
+    },
+    {
+      "epoch": 3.0,
+      "step": 8487,
+      "total_flos": 4467864361844736.0,
+      "train_loss": 8.53661620345823,
+      "train_runtime": 1479.3105,
+      "train_samples_per_second": 11.472,
+      "train_steps_per_second": 5.737
+    }
+  ],
+  "max_steps": 8487,
+  "num_train_epochs": 3,
+  "total_flos": 4467864361844736.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c148ad0d2e06033dd45e46c2206a01360d85aa367eec33748e3db7a87c0e86b
+size 3631