hpc model

README.md

@@ -1,3 +1,100 @@
 ---
+datasets:
+- BramVanroy/hebban-reviews
+language:
+- nl
 license: mit
+metrics:
+- accuracy
+- f1
+- precision
+- qwk
+- recall
+model-index:
+- name: robbert-v2-dutch-base-hebban-reviews5
+  results:
+  - dataset:
+      config: filtered_rating
+      name: BramVanroy/hebban-reviews - filtered_rating - 2.0.0
+      revision: 2.0.0
+      split: test
+      type: BramVanroy/hebban-reviews
+    metrics:
+    - name: Test accuracy
+      type: accuracy
+      value: 0.624457593688363
+    - name: Test f1
+      type: f1
+      value: 0.625518585787774
+    - name: Test precision
+      type: precision
+      value: 0.6295608657909847
+    - name: Test qwk
+      type: qwk
+      value: 0.7517620387343015
+    - name: Test recall
+      type: recall
+      value: 0.624457593688363
+    task:
+      name: sentiment analysis
+      type: text-classification
+tags:
+- sentiment-analysis
+- dutch
+- text
+widget:
+- text: Wauw, wat een leuk boek! Ik heb me er er goed mee vermaakt.
+- text: Nee, deze vond ik niet goed. De auteur doet zijn best om je als lezer mee
+    te trekken in het verhaal maar mij overtuigt het alleszins niet.
+- text: Ik vind het niet slecht maar de schrijfstijl trekt me ook niet echt aan. Het
+    wordt een beetje saai vanaf het vijfde hoofdstuk
 ---
+
+# robbert-v2-dutch-base-hebban-reviews
+
+# Dataset
+- dataset_name: BramVanroy/hebban-reviews
+- dataset_config: filtered_rating
+- dataset_revision: 2.0.0
+- labelcolumn: review_rating0
+- textcolumn: review_text_without_quotes
+
+# Training
+- optim: adamw_hf
+- learning_rate: 5e-05
+- per_device_train_batch_size: 64
+- per_device_eval_batch_size: 64
+- gradient_accumulation_steps: 1
+- max_steps: 5001
+- save_steps: 500
+- metric_for_best_model: qwk
+
+# Best checkedpoint based on validation
+- best_metric: 0.7480754124116261
+- best_model_checkpoint: trained/hebban-reviews5/robbert-v2-dutch-base/checkpoint-3000
+
+# Test results of best checkpoint
+- accuracy: 0.624457593688363
+- f1: 0.625518585787774
+- precision: 0.6295608657909847
+- qwk: 0.7517620387343015
+- recall: 0.624457593688363
+
+## Confusion matric
+
+![cfm](fig/test_confusion_matrix.png)
+
+## Normalized confusion matrix
+
+![norm cfm](fig/test_confusion_matrix_norm.png)
+
+# Environment
+- cuda_capabilities: 8.0; 8.0
+- cuda_device_count: 2
+- cuda_devices: NVIDIA A100-SXM4-80GB; NVIDIA A100-SXM4-80GB
+- finetuner_commit: 8159b4c1d5e66b36f68dd263299927ffb8670ebd
+- platform: Linux-4.18.0-305.49.1.el8_4.x86_64-x86_64-with-glibc2.28
+- python_version: 3.9.5
+- toch_version: 1.10.0
+- transformers_version: 4.21.0
+

all_results.json

@@ -0,0 +1,12 @@
+{
+    "accuracy": 0.624457593688363,
+    "epoch": 4.38,
+    "f1": 0.625518585787774,
+    "precision": 0.6295608657909847,
+    "qwk": 0.7517620387343015,
+    "recall": 0.624457593688363,
+    "train_loss": 0.7063156554363318,
+    "train_runtime": 2589.1522,
+    "train_samples_per_second": 247.235,
+    "train_steps_per_second": 1.932
+}

args.json

@@ -0,0 +1,136 @@
+{
+    "_n_gpu": 1,
+    "adafactor": false,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.999,
+    "adam_epsilon": 1e-08,
+    "auto_find_batch_size": false,
+    "bf16": false,
+    "bf16_full_eval": false,
+    "calculate_qwk": true,
+    "data_seed": 42,
+    "dataloader_drop_last": false,
+    "dataloader_num_workers": 0,
+    "dataloader_pin_memory": true,
+    "dataset_config": "filtered_rating",
+    "dataset_name": "BramVanroy/hebban-reviews",
+    "dataset_revision": "2.0.0",
+    "ddp_bucket_cap_mb": null,
+    "ddp_find_unused_parameters": null,
+    "debug": [],
+    "deepspeed": null,
+    "disable_tqdm": false,
+    "do_early_stopping": false,
+    "do_eval": true,
+    "do_optimize": false,
+    "do_predict": true,
+    "do_train": true,
+    "early_stopping_patience": 1,
+    "early_stopping_threshold": 0.0,
+    "eval_accumulation_steps": null,
+    "eval_delay": 0,
+    "eval_steps": 500,
+    "evaluation_strategy": "steps",
+    "fp16": true,
+    "fp16_backend": "auto",
+    "fp16_full_eval": false,
+    "fp16_opt_level": "O1",
+    "fsdp": [],
+    "fsdp_min_num_params": 0,
+    "fsdp_transformer_layer_cls_to_wrap": null,
+    "full_determinism": false,
+    "gradient_accumulation_steps": 1,
+    "gradient_checkpointing": false,
+    "greater_is_better": true,
+    "group_by_length": false,
+    "half_precision_backend": "cuda_amp",
+    "hub_model_id": null,
+    "hub_private_repo": false,
+    "hub_strategy": "every_save",
+    "hub_token": null,
+    "ignore_data_skip": false,
+    "include_inputs_for_metrics": false,
+    "jit_mode_eval": false,
+    "label_names": null,
+    "label_smoothing_factor": 0.0,
+    "labelcolumn": "review_rating0",
+    "labelnames": [
+        "1",
+        "2",
+        "3",
+        "4",
+        "5"
+    ],
+    "learning_rate": 5e-05,
+    "length_column_name": "length",
+    "load_best_model_at_end": true,
+    "local_rank": 0,
+    "log_level": -1,
+    "log_level_replica": -1,
+    "log_on_each_node": true,
+    "logging_dir": "trained/hebban-reviews5/robbert-v2-dutch-base/runs/Jul29_01-25-24_node3907.accelgor.os",
+    "logging_first_step": false,
+    "logging_nan_inf_filter": true,
+    "logging_steps": 500,
+    "logging_strategy": "steps",
+    "lr_scheduler_type": "linear",
+    "max_grad_norm": 1.0,
+    "max_seq_length": null,
+    "max_steps": 5001,
+    "max_test_samples": null,
+    "max_train_samples": null,
+    "max_validation_samples": null,
+    "metric_for_best_model": "qwk",
+    "model_name_or_path": "pdelobelle/robbert-v2-dutch-base",
+    "model_revision": "main",
+    "mp_parameters": "",
+    "n_trials": 8,
+    "no_cuda": false,
+    "num_train_epochs": 3.0,
+    "optim": "adamw_hf",
+    "output_dir": "trained/hebban-reviews5/robbert-v2-dutch-base",
+    "overwrite_cache": false,
+    "overwrite_output_dir": true,
+    "past_index": -1,
+    "per_device_eval_batch_size": 64,
+    "per_device_train_batch_size": 64,
+    "per_gpu_eval_batch_size": null,
+    "per_gpu_train_batch_size": null,
+    "prediction_loss_only": false,
+    "push_to_hub": false,
+    "push_to_hub_model_id": null,
+    "push_to_hub_organization": null,
+    "push_to_hub_token": null,
+    "ray_scope": "all",
+    "remove_unused_columns": true,
+    "report_to": [
+        "tensorboard"
+    ],
+    "resume_from_checkpoint": null,
+    "run_name": "trained/hebban-reviews5/robbert-v2-dutch-base",
+    "save_on_each_node": false,
+    "save_steps": 500,
+    "save_strategy": "steps",
+    "save_total_limit": null,
+    "scheduler_type": null,
+    "seed": 42,
+    "sharded_ddp": [],
+    "skip_memory_metrics": true,
+    "split_seed": 42,
+    "testsplit_name": "test",
+    "textcolumn": "review_text_without_quotes",
+    "tf32": null,
+    "torchdynamo": null,
+    "tpu_metrics_debug": false,
+    "tpu_num_cores": null,
+    "trainsplit_name": "train",
+    "use_class_weights": true,
+    "use_ipex": false,
+    "use_legacy_prediction_loop": false,
+    "validation_size": 0.1,
+    "validationsplit_name": "validation",
+    "warmup_ratio": 0.0,
+    "warmup_steps": 0,
+    "weight_decay": 0.0,
+    "xpu_backend": null
+}

config.json

@@ -0,0 +1,43 @@
+{
+  "_name_or_path": "pdelobelle/robbert-v2-dutch-base",
+  "architectures": [
+    "RobertaForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "1",
+    "1": "2",
+    "2": "3",
+    "3": "4",
+    "4": "5"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "1": 0,
+    "2": 1,
+    "3": 2,
+    "4": 3,
+    "5": 4
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 40000
+}

env.json

@@ -0,0 +1,10 @@
+{
+    "cuda_capabilities": "8.0; 8.0",
+    "cuda_device_count": 2,
+    "cuda_devices": "NVIDIA A100-SXM4-80GB; NVIDIA A100-SXM4-80GB",
+    "finetuner_commit": "8159b4c1d5e66b36f68dd263299927ffb8670ebd",
+    "platform": "Linux-4.18.0-305.49.1.el8_4.x86_64-x86_64-with-glibc2.28",
+    "python_version": "3.9.5",
+    "toch_version": "1.10.0",
+    "transformers_version": "4.21.0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:859f9339b67455c3bdef70d80ee6f6d14dce44c60ec4c6bb81e08ef8a7f8c9f5
+size 467135469

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

test_predictions.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:912d8eacf7f6c8db3430c5587308096308cef03e1d555ce036ce91babad3187b
+size 65128043

test_results.json

@@ -0,0 +1,7 @@
+{
+    "accuracy": 0.624457593688363,
+    "f1": 0.625518585787774,
+    "precision": 0.6295608657909847,
+    "qwk": 0.7517620387343015,
+    "recall": 0.624457593688363
+}

tokenizer_config.json

@@ -0,0 +1,65 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "pdelobelle/robbert-v2-dutch-base",
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "special_tokens_map_file": "./robbert-v2-dutch-base/special_tokens_map.json",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 4.38,
+    "train_loss": 0.7063156554363318,
+    "train_runtime": 2589.1522,
+    "train_samples_per_second": 247.235,
+    "train_steps_per_second": 1.932
+}

trainer_state.json

@@ -0,0 +1,215 @@
+{
+  "best_metric": 0.7480754124116261,
+  "best_model_checkpoint": "trained/hebban-reviews5/robbert-v2-dutch-base/checkpoint-3000",
+  "epoch": 4.382997370727432,
+  "global_step": 5001,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.44,
+      "learning_rate": 4.501099780043991e-05,
+      "loss": 1.0492,
+      "step": 500
+    },
+    {
+      "epoch": 0.44,
+      "eval_accuracy": 0.5691568047337278,
+      "eval_f1": 0.5636413822079555,
+      "eval_loss": 0.9223524332046509,
+      "eval_precision": 0.5918143879920826,
+      "eval_qwk": 0.7063098801003743,
+      "eval_recall": 0.5691568047337278,
+      "eval_runtime": 23.2899,
+      "eval_samples_per_second": 696.61,
+      "eval_steps_per_second": 5.453,
+      "step": 500
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 4.0011997600479906e-05,
+      "loss": 0.9178,
+      "step": 1000
+    },
+    {
+      "epoch": 0.88,
+      "eval_accuracy": 0.5837031558185405,
+      "eval_f1": 0.5793159803831781,
+      "eval_loss": 0.9023029208183289,
+      "eval_precision": 0.6074918212756761,
+      "eval_qwk": 0.7007746526525567,
+      "eval_recall": 0.5837031558185405,
+      "eval_runtime": 23.0813,
+      "eval_samples_per_second": 702.906,
+      "eval_steps_per_second": 5.502,
+      "step": 1000
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 3.5022995400919816e-05,
+      "loss": 0.8109,
+      "step": 1500
+    },
+    {
+      "epoch": 1.31,
+      "eval_accuracy": 0.5977564102564102,
+      "eval_f1": 0.5928304451541198,
+      "eval_loss": 0.9542309641838074,
+      "eval_precision": 0.61219538907447,
+      "eval_qwk": 0.7336756182944968,
+      "eval_recall": 0.5977564102564102,
+      "eval_runtime": 22.9709,
+      "eval_samples_per_second": 706.285,
+      "eval_steps_per_second": 5.529,
+      "step": 1500
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 3.0033993201359727e-05,
+      "loss": 0.7808,
+      "step": 2000
+    },
+    {
+      "epoch": 1.75,
+      "eval_accuracy": 0.6158777120315582,
+      "eval_f1": 0.6160589696416037,
+      "eval_loss": 0.9172691106796265,
+      "eval_precision": 0.6237298170154965,
+      "eval_qwk": 0.7443852226981237,
+      "eval_recall": 0.6158777120315582,
+      "eval_runtime": 22.9508,
+      "eval_samples_per_second": 706.903,
+      "eval_steps_per_second": 5.534,
+      "step": 2000
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 2.503499300139972e-05,
+      "loss": 0.704,
+      "step": 2500
+    },
+    {
+      "epoch": 2.19,
+      "eval_accuracy": 0.6121794871794872,
+      "eval_f1": 0.6079206553629232,
+      "eval_loss": 1.1115827560424805,
+      "eval_precision": 0.6233537730211096,
+      "eval_qwk": 0.7381725383149609,
+      "eval_recall": 0.6121794871794872,
+      "eval_runtime": 23.189,
+      "eval_samples_per_second": 699.641,
+      "eval_steps_per_second": 5.477,
+      "step": 2500
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 2.0045990801839633e-05,
+      "loss": 0.6401,
+      "step": 3000
+    },
+    {
+      "epoch": 2.63,
+      "eval_accuracy": 0.6272805719921104,
+      "eval_f1": 0.6283777941836641,
+      "eval_loss": 1.1353460550308228,
+      "eval_precision": 0.6325852588723305,
+      "eval_qwk": 0.7480754124116261,
+      "eval_recall": 0.6272805719921104,
+      "eval_runtime": 23.0847,
+      "eval_samples_per_second": 702.804,
+      "eval_steps_per_second": 5.501,
+      "step": 3000
+    },
+    {
+      "epoch": 3.07,
+      "learning_rate": 1.5046990601879624e-05,
+      "loss": 0.612,
+      "step": 3500
+    },
+    {
+      "epoch": 3.07,
+      "eval_accuracy": 0.624198717948718,
+      "eval_f1": 0.6228264241003357,
+      "eval_loss": 1.2358475923538208,
+      "eval_precision": 0.6296279459229124,
+      "eval_qwk": 0.7443813858149255,
+      "eval_recall": 0.624198717948718,
+      "eval_runtime": 23.07,
+      "eval_samples_per_second": 703.252,
+      "eval_steps_per_second": 5.505,
+      "step": 3500
+    },
+    {
+      "epoch": 3.51,
+      "learning_rate": 1.0047990401919617e-05,
+      "loss": 0.536,
+      "step": 4000
+    },
+    {
+      "epoch": 3.51,
+      "eval_accuracy": 0.6244452662721893,
+      "eval_f1": 0.623840669755937,
+      "eval_loss": 1.2645397186279297,
+      "eval_precision": 0.6306447854320058,
+      "eval_qwk": 0.743683140105029,
+      "eval_recall": 0.6244452662721893,
+      "eval_runtime": 22.815,
+      "eval_samples_per_second": 711.112,
+      "eval_steps_per_second": 5.567,
+      "step": 4000
+    },
+    {
+      "epoch": 3.94,
+      "learning_rate": 5.048990201959608e-06,
+      "loss": 0.5326,
+      "step": 4500
+    },
+    {
+      "epoch": 3.94,
+      "eval_accuracy": 0.6256780078895463,
+      "eval_f1": 0.625250776842927,
+      "eval_loss": 1.2848360538482666,
+      "eval_precision": 0.6314145408376363,
+      "eval_qwk": 0.7451057727705823,
+      "eval_recall": 0.6256780078895463,
+      "eval_runtime": 23.0764,
+      "eval_samples_per_second": 703.056,
+      "eval_steps_per_second": 5.503,
+      "step": 4500
+    },
+    {
+      "epoch": 4.38,
+      "learning_rate": 4.999000199960008e-08,
+      "loss": 0.4802,
+      "step": 5000
+    },
+    {
+      "epoch": 4.38,
+      "eval_accuracy": 0.6283900394477318,
+      "eval_f1": 0.6282900923952147,
+      "eval_loss": 1.3624160289764404,
+      "eval_precision": 0.6318538074407878,
+      "eval_qwk": 0.7462804704035614,
+      "eval_recall": 0.6283900394477318,
+      "eval_runtime": 23.1033,
+      "eval_samples_per_second": 702.237,
+      "eval_steps_per_second": 5.497,
+      "step": 5000
+    },
+    {
+      "epoch": 4.38,
+      "step": 5001,
+      "total_flos": 1.6839139993111757e+17,
+      "train_loss": 0.7063156554363318,
+      "train_runtime": 2589.1522,
+      "train_samples_per_second": 247.235,
+      "train_steps_per_second": 1.932
+    }
+  ],
+  "max_steps": 5001,
+  "num_train_epochs": 5,
+  "total_flos": 1.6839139993111757e+17,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e86810b60f2ada4bf4d6488355a955e5351bc5ac67c2be4187afbc2f1b7cc3b4
+size 3375