add model

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

README.md

@@ -0,0 +1,90 @@
+---
+license: apache-2.0
+tags:
+- generated_from_trainer
+metrics:
+- rouge
+model-index:
+- name: barthez-deft-chimie
+  results:
+  - task:
+      name: Summarization
+      type: summarization
+    metrics:
+    - name: Rouge1
+      type: rouge
+      value: 31.8947
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# barthez-deft-chimie
+
+This model is a fine-tuned version of [moussaKam/barthez](https://huggingface.co/moussaKam/barthez) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.0710
+- Rouge1: 31.8947
+- Rouge2: 16.7563
+- Rougel: 23.5428
+- Rougelsum: 23.4918
+- Gen Len: 38.5256
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 20.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rouge1  | Rouge2  | Rougel  | Rougelsum | Gen Len |
+|:-------------:|:-----:|:----:|:---------------:|:-------:|:-------:|:-------:|:---------:|:-------:|
+| 3.8022        | 1.0   | 118  | 2.5491          | 16.8208 | 7.0027  | 13.957  | 14.0479   | 19.1538 |
+| 2.9286        | 2.0   | 236  | 2.3074          | 17.5356 | 7.8717  | 14.4874 | 14.5044   | 19.9487 |
+| 2.5422        | 3.0   | 354  | 2.2322          | 19.6491 | 9.4156  | 15.9467 | 15.9433   | 19.7051 |
+| 2.398         | 4.0   | 472  | 2.1500          | 18.7166 | 9.859   | 15.7535 | 15.8036   | 19.9231 |
+| 2.2044        | 5.0   | 590  | 2.1372          | 19.978  | 10.6235 | 16.1348 | 16.1274   | 19.6154 |
+| 1.9405        | 6.0   | 708  | 2.0992          | 20.226  | 10.551  | 16.6928 | 16.7211   | 19.9744 |
+| 1.8544        | 7.0   | 826  | 2.0841          | 19.8869 | 10.8456 | 16.1072 | 16.097    | 19.8846 |
+| 1.7536        | 8.0   | 944  | 2.0791          | 19.3017 | 9.4921  | 16.1541 | 16.2167   | 19.859  |
+| 1.6914        | 9.0   | 1062 | 2.0710          | 21.3848 | 10.4088 | 17.1963 | 17.2254   | 19.8846 |
+| 1.654         | 10.0  | 1180 | 2.1069          | 22.3811 | 10.7987 | 18.7595 | 18.761    | 19.9231 |
+| 1.5899        | 11.0  | 1298 | 2.0919          | 20.8546 | 10.6958 | 16.8637 | 16.9499   | 19.8077 |
+| 1.4661        | 12.0  | 1416 | 2.1065          | 22.3677 | 11.7472 | 18.262  | 18.3      | 19.9744 |
+| 1.4205        | 13.0  | 1534 | 2.1164          | 20.5845 | 10.7825 | 16.9972 | 17.0216   | 19.9359 |
+| 1.3797        | 14.0  | 1652 | 2.1240          | 22.2561 | 11.303  | 17.5064 | 17.5815   | 19.9744 |
+| 1.3724        | 15.0  | 1770 | 2.1187          | 23.2825 | 11.912  | 18.5208 | 18.5499   | 19.9359 |
+| 1.3404        | 16.0  | 1888 | 2.1394          | 22.1305 | 10.5258 | 17.772  | 17.8202   | 19.9744 |
+| 1.2846        | 17.0  | 2006 | 2.1502          | 21.567  | 11.0557 | 17.2562 | 17.2974   | 20.0    |
+| 1.2871        | 18.0  | 2124 | 2.1572          | 22.5871 | 11.702  | 18.2906 | 18.3826   | 19.9744 |
+| 1.2422        | 19.0  | 2242 | 2.1613          | 23.0935 | 11.6824 | 18.6087 | 18.6777   | 19.9744 |
+| 1.2336        | 20.0  | 2360 | 2.1581          | 22.6789 | 11.4363 | 18.1661 | 18.2346   | 19.9487 |
+
+
+### Framework versions
+
+- Transformers 4.10.2
+- Pytorch 1.7.1+cu110
+- Datasets 1.11.0
+- Tokenizers 0.10.3

all_results.json

@@ -0,0 +1,18 @@
+{
+    "epoch": 20.0,
+    "eval_gen_len": 38.5256,
+    "eval_loss": 2.0710394382476807,
+    "eval_rouge1": 31.8947,
+    "eval_rouge2": 16.7563,
+    "eval_rougeL": 23.5428,
+    "eval_rougeLsum": 23.4918,
+    "eval_runtime": 12.584,
+    "eval_samples": 78,
+    "eval_samples_per_second": 6.198,
+    "eval_steps_per_second": 1.589,
+    "train_loss": 1.7793180174746757,
+    "train_runtime": 373.3963,
+    "train_samples": 469,
+    "train_samples_per_second": 25.121,
+    "train_steps_per_second": 6.32
+}

config.json

@@ -0,0 +1,59 @@
+{
+  "_name_or_path": "moussaKam/barthez",
+  "activation_dropout": 0.1,
+  "activation_function": "gelu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": true,
+  "architectures": [
+    "MBartForConditionalGeneration"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "classif_dropout": 0.1,
+  "classifier_dropout": 0.0,
+  "d_model": 768,
+  "decoder_attention_heads": 12,
+  "decoder_ffn_dim": 3072,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 2,
+  "do_blenderbot_90_layernorm": false,
+  "dropout": 0.1,
+  "early_stopping": true,
+  "encoder_attention_heads": 12,
+  "encoder_ffn_dim": 3072,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "eos_token_id": 2,
+  "extra_pos_embeddings": 2,
+  "force_bos_token_to_be_generated": false,
+  "forced_eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_position_embeddings": 1024,
+  "model_type": "mbart",
+  "no_repeat_ngram_size": 3,
+  "normalize_before": true,
+  "normalize_embedding": true,
+  "num_beams": 4,
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "scale_embedding": false,
+  "static_position_embeddings": false,
+  "tokenizer_class": "BarthezTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.10.2",
+  "use_cache": true,
+  "vocab_size": 50002
+}

eval_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 20.0,
+    "eval_gen_len": 38.5256,
+    "eval_loss": 2.0710394382476807,
+    "eval_rouge1": 31.8947,
+    "eval_rouge2": 16.7563,
+    "eval_rougeL": 23.5428,
+    "eval_rougeLsum": 23.4918,
+    "eval_runtime": 12.584,
+    "eval_samples": 78,
+    "eval_samples_per_second": 6.198,
+    "eval_steps_per_second": 1.589
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3560635a9a51a7660e667116ae639ec221e0a0d7834203b2cac7eb9417e73660
+size 557190519

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:582403da7cb525c4cac90b495e553cb766b818be71755e6bd885072a1067314a
+size 1115393

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "moussaKam/barthez", "tokenizer_class": "BarthezTokenizer"}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 20.0,
+    "train_loss": 1.7793180174746757,
+    "train_runtime": 373.3963,
+    "train_samples": 469,
+    "train_samples_per_second": 25.121,
+    "train_steps_per_second": 6.32
+}

trainer_state.json

@@ -0,0 +1,423 @@
+{
+  "best_metric": 2.0710394382476807,
+  "best_model_checkpoint": "./checkpoints/barthez-deft-chimie/checkpoint-1062",
+  "epoch": 20.0,
+  "global_step": 2360,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.85,
+      "learning_rate": 2.8805084745762714e-05,
+      "loss": 3.8022,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_gen_len": 19.1538,
+      "eval_loss": 2.5490684509277344,
+      "eval_rouge1": 16.8208,
+      "eval_rouge2": 7.0027,
+      "eval_rougeL": 13.957,
+      "eval_rougeLsum": 14.0479,
+      "eval_runtime": 3.9642,
+      "eval_samples_per_second": 19.676,
+      "eval_steps_per_second": 5.045,
+      "step": 118
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 2.7533898305084746e-05,
+      "loss": 2.9286,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_gen_len": 19.9487,
+      "eval_loss": 2.3073835372924805,
+      "eval_rouge1": 17.5356,
+      "eval_rouge2": 7.8717,
+      "eval_rougeL": 14.4874,
+      "eval_rougeLsum": 14.5044,
+      "eval_runtime": 3.9919,
+      "eval_samples_per_second": 19.54,
+      "eval_steps_per_second": 5.01,
+      "step": 236
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 2.627542372881356e-05,
+      "loss": 2.5422,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_gen_len": 19.7051,
+      "eval_loss": 2.232180118560791,
+      "eval_rouge1": 19.6491,
+      "eval_rouge2": 9.4156,
+      "eval_rougeL": 15.9467,
+      "eval_rougeLsum": 15.9433,
+      "eval_runtime": 3.9423,
+      "eval_samples_per_second": 19.785,
+      "eval_steps_per_second": 5.073,
+      "step": 354
+    },
+    {
+      "epoch": 3.39,
+      "learning_rate": 2.5004237288135596e-05,
+      "loss": 2.398,
+      "step": 400
+    },
+    {
+      "epoch": 4.0,
+      "eval_gen_len": 19.9231,
+      "eval_loss": 2.150047540664673,
+      "eval_rouge1": 18.7166,
+      "eval_rouge2": 9.859,
+      "eval_rougeL": 15.7535,
+      "eval_rougeLsum": 15.8036,
+      "eval_runtime": 3.904,
+      "eval_samples_per_second": 19.98,
+      "eval_steps_per_second": 5.123,
+      "step": 472
+    },
+    {
+      "epoch": 4.24,
+      "learning_rate": 2.373305084745763e-05,
+      "loss": 2.2044,
+      "step": 500
+    },
+    {
+      "epoch": 5.0,
+      "eval_gen_len": 19.6154,
+      "eval_loss": 2.1371994018554688,
+      "eval_rouge1": 19.978,
+      "eval_rouge2": 10.6235,
+      "eval_rougeL": 16.1348,
+      "eval_rougeLsum": 16.1274,
+      "eval_runtime": 3.9032,
+      "eval_samples_per_second": 19.984,
+      "eval_steps_per_second": 5.124,
+      "step": 590
+    },
+    {
+      "epoch": 5.08,
+      "learning_rate": 2.2461864406779662e-05,
+      "loss": 2.0419,
+      "step": 600
+    },
+    {
+      "epoch": 5.93,
+      "learning_rate": 2.1203389830508474e-05,
+      "loss": 1.9405,
+      "step": 700
+    },
+    {
+      "epoch": 6.0,
+      "eval_gen_len": 19.9744,
+      "eval_loss": 2.099212884902954,
+      "eval_rouge1": 20.226,
+      "eval_rouge2": 10.551,
+      "eval_rougeL": 16.6928,
+      "eval_rougeLsum": 16.7211,
+      "eval_runtime": 3.9012,
+      "eval_samples_per_second": 19.994,
+      "eval_steps_per_second": 5.127,
+      "step": 708
+    },
+    {
+      "epoch": 6.78,
+      "learning_rate": 1.993220338983051e-05,
+      "loss": 1.8544,
+      "step": 800
+    },
+    {
+      "epoch": 7.0,
+      "eval_gen_len": 19.8846,
+      "eval_loss": 2.0841410160064697,
+      "eval_rouge1": 19.8869,
+      "eval_rouge2": 10.8456,
+      "eval_rougeL": 16.1072,
+      "eval_rougeLsum": 16.097,
+      "eval_runtime": 3.8835,
+      "eval_samples_per_second": 20.085,
+      "eval_steps_per_second": 5.15,
+      "step": 826
+    },
+    {
+      "epoch": 7.63,
+      "learning_rate": 1.866101694915254e-05,
+      "loss": 1.7536,
+      "step": 900
+    },
+    {
+      "epoch": 8.0,
+      "eval_gen_len": 19.859,
+      "eval_loss": 2.0790863037109375,
+      "eval_rouge1": 19.3017,
+      "eval_rouge2": 9.4921,
+      "eval_rougeL": 16.1541,
+      "eval_rougeLsum": 16.2167,
+      "eval_runtime": 3.9024,
+      "eval_samples_per_second": 19.988,
+      "eval_steps_per_second": 5.125,
+      "step": 944
+    },
+    {
+      "epoch": 8.47,
+      "learning_rate": 1.7389830508474576e-05,
+      "loss": 1.6914,
+      "step": 1000
+    },
+    {
+      "epoch": 9.0,
+      "eval_gen_len": 19.8846,
+      "eval_loss": 2.0710394382476807,
+      "eval_rouge1": 21.3848,
+      "eval_rouge2": 10.4088,
+      "eval_rougeL": 17.1963,
+      "eval_rougeLsum": 17.2254,
+      "eval_runtime": 3.8762,
+      "eval_samples_per_second": 20.123,
+      "eval_steps_per_second": 5.16,
+      "step": 1062
+    },
+    {
+      "epoch": 9.32,
+      "learning_rate": 1.611864406779661e-05,
+      "loss": 1.654,
+      "step": 1100
+    },
+    {
+      "epoch": 10.0,
+      "eval_gen_len": 19.9231,
+      "eval_loss": 2.106861114501953,
+      "eval_rouge1": 22.3811,
+      "eval_rouge2": 10.7987,
+      "eval_rougeL": 18.7595,
+      "eval_rougeLsum": 18.761,
+      "eval_runtime": 3.9276,
+      "eval_samples_per_second": 19.859,
+      "eval_steps_per_second": 5.092,
+      "step": 1180
+    },
+    {
+      "epoch": 10.17,
+      "learning_rate": 1.4847457627118646e-05,
+      "loss": 1.5899,
+      "step": 1200
+    },
+    {
+      "epoch": 11.0,
+      "eval_gen_len": 19.8077,
+      "eval_loss": 2.09187650680542,
+      "eval_rouge1": 20.8546,
+      "eval_rouge2": 10.6958,
+      "eval_rougeL": 16.8637,
+      "eval_rougeLsum": 16.9499,
+      "eval_runtime": 3.8954,
+      "eval_samples_per_second": 20.024,
+      "eval_steps_per_second": 5.134,
+      "step": 1298
+    },
+    {
+      "epoch": 11.02,
+      "learning_rate": 1.357627118644068e-05,
+      "loss": 1.534,
+      "step": 1300
+    },
+    {
+      "epoch": 11.86,
+      "learning_rate": 1.2305084745762711e-05,
+      "loss": 1.4661,
+      "step": 1400
+    },
+    {
+      "epoch": 12.0,
+      "eval_gen_len": 19.9744,
+      "eval_loss": 2.106508255004883,
+      "eval_rouge1": 22.3677,
+      "eval_rouge2": 11.7472,
+      "eval_rougeL": 18.262,
+      "eval_rougeLsum": 18.3,
+      "eval_runtime": 3.8544,
+      "eval_samples_per_second": 20.237,
+      "eval_steps_per_second": 5.189,
+      "step": 1416
+    },
+    {
+      "epoch": 12.71,
+      "learning_rate": 1.1033898305084746e-05,
+      "loss": 1.4205,
+      "step": 1500
+    },
+    {
+      "epoch": 13.0,
+      "eval_gen_len": 19.9359,
+      "eval_loss": 2.116386651992798,
+      "eval_rouge1": 20.5845,
+      "eval_rouge2": 10.7825,
+      "eval_rougeL": 16.9972,
+      "eval_rougeLsum": 17.0216,
+      "eval_runtime": 3.9246,
+      "eval_samples_per_second": 19.875,
+      "eval_steps_per_second": 5.096,
+      "step": 1534
+    },
+    {
+      "epoch": 13.56,
+      "learning_rate": 9.76271186440678e-06,
+      "loss": 1.3797,
+      "step": 1600
+    },
+    {
+      "epoch": 14.0,
+      "eval_gen_len": 19.9744,
+      "eval_loss": 2.124025344848633,
+      "eval_rouge1": 22.2561,
+      "eval_rouge2": 11.303,
+      "eval_rougeL": 17.5064,
+      "eval_rougeLsum": 17.5815,
+      "eval_runtime": 3.8828,
+      "eval_samples_per_second": 20.089,
+      "eval_steps_per_second": 5.151,
+      "step": 1652
+    },
+    {
+      "epoch": 14.41,
+      "learning_rate": 8.491525423728813e-06,
+      "loss": 1.3724,
+      "step": 1700
+    },
+    {
+      "epoch": 15.0,
+      "eval_gen_len": 19.9359,
+      "eval_loss": 2.118665933609009,
+      "eval_rouge1": 23.2825,
+      "eval_rouge2": 11.912,
+      "eval_rougeL": 18.5208,
+      "eval_rougeLsum": 18.5499,
+      "eval_runtime": 3.9164,
+      "eval_samples_per_second": 19.916,
+      "eval_steps_per_second": 5.107,
+      "step": 1770
+    },
+    {
+      "epoch": 15.25,
+      "learning_rate": 7.220338983050848e-06,
+      "loss": 1.3404,
+      "step": 1800
+    },
+    {
+      "epoch": 16.0,
+      "eval_gen_len": 19.9744,
+      "eval_loss": 2.139371395111084,
+      "eval_rouge1": 22.1305,
+      "eval_rouge2": 10.5258,
+      "eval_rougeL": 17.772,
+      "eval_rougeLsum": 17.8202,
+      "eval_runtime": 3.9181,
+      "eval_samples_per_second": 19.908,
+      "eval_steps_per_second": 5.105,
+      "step": 1888
+    },
+    {
+      "epoch": 16.1,
+      "learning_rate": 5.949152542372881e-06,
+      "loss": 1.2967,
+      "step": 1900
+    },
+    {
+      "epoch": 16.95,
+      "learning_rate": 4.677966101694916e-06,
+      "loss": 1.2846,
+      "step": 2000
+    },
+    {
+      "epoch": 17.0,
+      "eval_gen_len": 20.0,
+      "eval_loss": 2.1501927375793457,
+      "eval_rouge1": 21.567,
+      "eval_rouge2": 11.0557,
+      "eval_rougeL": 17.2562,
+      "eval_rougeLsum": 17.2974,
+      "eval_runtime": 3.8698,
+      "eval_samples_per_second": 20.156,
+      "eval_steps_per_second": 5.168,
+      "step": 2006
+    },
+    {
+      "epoch": 17.8,
+      "learning_rate": 3.406779661016949e-06,
+      "loss": 1.2871,
+      "step": 2100
+    },
+    {
+      "epoch": 18.0,
+      "eval_gen_len": 19.9744,
+      "eval_loss": 2.1571571826934814,
+      "eval_rouge1": 22.5871,
+      "eval_rouge2": 11.702,
+      "eval_rougeL": 18.2906,
+      "eval_rougeLsum": 18.3826,
+      "eval_runtime": 3.8649,
+      "eval_samples_per_second": 20.182,
+      "eval_steps_per_second": 5.175,
+      "step": 2124
+    },
+    {
+      "epoch": 18.64,
+      "learning_rate": 2.1355932203389833e-06,
+      "loss": 1.2422,
+      "step": 2200
+    },
+    {
+      "epoch": 19.0,
+      "eval_gen_len": 19.9744,
+      "eval_loss": 2.161255359649658,
+      "eval_rouge1": 23.0935,
+      "eval_rouge2": 11.6824,
+      "eval_rougeL": 18.6087,
+      "eval_rougeLsum": 18.6777,
+      "eval_runtime": 3.8614,
+      "eval_samples_per_second": 20.2,
+      "eval_steps_per_second": 5.18,
+      "step": 2242
+    },
+    {
+      "epoch": 19.49,
+      "learning_rate": 8.64406779661017e-07,
+      "loss": 1.2336,
+      "step": 2300
+    },
+    {
+      "epoch": 20.0,
+      "eval_gen_len": 19.9487,
+      "eval_loss": 2.158088445663452,
+      "eval_rouge1": 22.6789,
+      "eval_rouge2": 11.4363,
+      "eval_rougeL": 18.1661,
+      "eval_rougeLsum": 18.2346,
+      "eval_runtime": 3.8624,
+      "eval_samples_per_second": 20.195,
+      "eval_steps_per_second": 5.178,
+      "step": 2360
+    },
+    {
+      "epoch": 20.0,
+      "step": 2360,
+      "total_flos": 1837511390490624.0,
+      "train_loss": 1.7793180174746757,
+      "train_runtime": 373.3963,
+      "train_samples_per_second": 25.121,
+      "train_steps_per_second": 6.32
+    }
+  ],
+  "max_steps": 2360,
+  "num_train_epochs": 20,
+  "total_flos": 1837511390490624.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b366072883e294bb66de9dd12116fe6df2dcf5fbefdd2a9e1e596150b45f5d2
+size 2863