Initial commit

.gitattributes

@@ -0,0 +1,4 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+runs/** filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,128 @@
+---
+language:
+  - de
+tags:
+  - question-generation
+  - german
+  - text2text-generation
+  - generated_from_trainer
+datasets:
+  - lmqg/qg_dequad
+metrics:
+  - bleu4
+  - f1
+  - rouge
+  - exact_match
+model-index:
+  - name: german-jeopardy-mt5-base-256
+    results:
+      - task:
+          name: Sequence-to-sequence Language Modeling
+          type: text2text-generation
+        dataset:
+          name: lmqg/qg_dequad
+          type: default
+          args: default
+        metrics:
+          - name: BLEU-4
+            type: bleu4
+            value: 13.70
+          - name: F1
+            type: f1
+            value: 37.79
+          - name: ROUGE-1
+            type: rouge1
+            value: 38.80
+          - name: ROUGE-2
+            type: rouge2
+            value: 20.27
+          - name: ROUGE-L
+            type: rougel
+            value: 37.34
+          - name: ROUGE-Lsum
+            type: rougelsum
+            value: 37.32
+          - name: Exact Match
+            type: exact_match
+            value: 2.81
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# german-jeopardy-mt5-base-256
+
+This model is a fine-tuned version of [google/mt5-base](https://huggingface.co/google/mt5-base) on the [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad) dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.51
+- Brevity Penalty: 0.8658
+- System Length: 18174
+- Reference Length: 20793
+- ROUGE-1: 38.80
+- ROUGE-2: 20.27
+- ROUGE-L: 37.34
+- ROUGE-Lsum: 37.32
+- Exact Match: 2.81
+- BLEU: 13.70
+- F1: 37.79
+
+## Model description
+
+See [google/mt5-base](https://huggingface.co/google/mt5-base) for the model architecture.  
+The model was trained on a single NVIDIA RTX 3090 GPU with 24GB of VRAM.
+
+## Intended uses & limitations
+
+This model can be used for question generation on German text.
+
+## Training and evaluation data
+
+See [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad).
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 7
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 256
+- optimizer: Adafactor
+- lr_scheduler_type: constant
+- num_epochs: 20
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Counts 1 | Counts 2 | Counts 3 | Counts 4 | Totals 1 | Totals 2 | Totals 3 | Totals 4 | Precisions 1 | Precisions 2 | Precisions 3 | Precisions 4 | Brevity Penalty | System Length | Reference Length | ROUGE-1 | ROUGE-2 | ROUGE-L | ROUGE-Lsum | Exact Match |  BLEU   | Mean Generated Length |   F1   |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:------------:|:------------:|:------------:|:------------:|:---------------:|:-------------:|:----------------:|:-------:|:-------:|:-------:|:----------:|:-----------:|:-------:|:---------------------:|:------:|
+|    8.9608     | 0.99  |  36  |     2.8883      |   2306   |    50    |    12    |    2     |  17876   |  15672   |  13468   |  11264   |     12.9     |    0.319     |    0.0891    |    0.0178    |      0.828      |     17876     |      21250       | 0.0081  | 0.0022  | 0.0078  |   0.0078   |     0.0     | 0.2352  |        3.1969         | 0.0092 |
+|    3.2364     | 1.98  |  72  |     1.9242      |   6125   |   1727   |   687    |   277    |  21152   |  18948   |  16744   |  14540   |   28.9571    |    9.1144    |    4.103     |    1.9051    |     0.9954      |     21152     |      21250       | 0.2457  | 0.1026  | 0.2345  |   0.2346   |   0.0018    | 6.7083  |        11.8072        | 0.2514 |
+|    2.4963     |  3.0  | 109  |     1.6558      |   6903   |   2271   |   975    |   409    |  16537   |  14333   |  12129   |   9925   |   41.7428    |   15.8446    |    8.0386    |    4.1209    |      0.752      |     16537     |      21250       | 0.2966  | 0.1415  | 0.2854  |   0.2852   |    0.01     | 9.1493  |        12.176         | 0.2909 |
+|    2.2314     | 3.98  | 145  |     1.5771      |   7160   |   2440   |   1098   |   501    |  16627   |  14423   |  12219   |  10015   |   43.0625    |   16.9174    |    8.986     |    5.0025    |     0.7573      |     16627     |      21250       |  0.314  | 0.1535  | 0.3028  |   0.3028   |   0.0136    | 10.187  |        12.157         | 0.3069 |
+|    2.0578     | 4.97  | 181  |     1.5347      |   7447   |   2625   |   1214   |   566    |  17305   |  15101   |  12897   |  10693   |   43.0338    |    17.383    |    9.413     |    5.2932    |     0.7961      |     17305     |      21250       | 0.3286  | 0.1628  | 0.3146  |   0.3146   |   0.0163    | 11.0621 |        12.5585        |  0.32  |
+|    1.8928     | 5.99  | 218  |     1.5128      |   7396   |   2659   |   1257   |   611    |  16598   |  14394   |  12190   |   9986   |   44.5596    |    18.473    |   10.3117    |    6.1186    |     0.7556      |     16598     |      21250       | 0.3326  | 0.1684  | 0.3198  |   0.3198   |   0.0177    | 11.4063 |        12.1692        | 0.3234 |
+|    1.8573     | 6.98  | 254  |     1.4736      |   7531   |   2758   |   1313   |   641    |  16728   |  14524   |  12320   |  10116   |   45.0203    |   18.9893    |   10.6575    |    6.3365    |     0.7631      |     16728     |      21250       | 0.3349  | 0.1717  | 0.3216  |   0.3216   |   0.0163    | 11.8292 |        12.3035        | 0.327  |
+|    1.7361     |  8.0  | 291  |     1.4544      |   7658   |   2849   |   1368   |   668    |  16928   |  14724   |  12520   |  10316   |   45.2387    |   19.3494    |   10.9265    |    6.4754    |     0.7747      |     16928     |      21250       | 0.3414  | 0.1762  | 0.3283  |   0.3284   |   0.0181    | 12.2208 |        12.4628        | 0.3334 |
+|    1.7162     | 8.99  | 327  |     1.4459      |   7703   |   2891   |   1390   |   694    |  16795   |  14591   |  12387   |  10183   |   45.8648    |   19.8136    |   11.2214    |    6.8153    |      0.767      |     16795     |      21250       | 0.3454  | 0.1785  | 0.3325  |   0.3323   |   0.0159    | 12.4536 |        12.4174        | 0.3374 |
+|    1.6589     | 9.98  | 363  |     1.4383      |   7889   |   2983   |   1449   |   719    |  17376   |  15172   |  12968   |  10764   |   45.4017    |   19.6612    |   11.1737    |    6.6797    |     0.8002      |     17376     |      21250       | 0.3519  | 0.1816  | 0.3375  |   0.3372   |   0.0172    | 12.8553 |        12.7101        | 0.3435 |
+|    1.5571     | 10.99 | 400  |     1.4214      |   7889   |   2994   |   1457   |   736    |  17185   |  14981   |  12777   |  10573   |   45.9063    |   19.9853    |   11.4033    |    6.9611    |     0.7894      |     17185     |      21250       | 0.3529  | 0.1845  | 0.3392  |   0.3393   |    0.02     | 12.9671 |        12.6466        | 0.3457 |
+|    1.5502     | 11.98 | 436  |     1.4135      |   7930   |   3008   |   1477   |   741    |  16868   |  14664   |  12460   |  10256   |   47.0121    |   20.5128    |   11.8539    |    7.225     |     0.7712      |     16868     |      21250       | 0.3619  |  0.189  | 0.3492  |   0.3491   |   0.0213    | 13.0741 |        12.4483        | 0.3541 |
+|    1.4564     | 13.0  | 473  |     1.3943      |   8268   |   3200   |   1616   |   837    |  17929   |  15725   |  13521   |  11317   |   46.1152    |   20.3498    |   11.9518    |    7.396     |     0.8309      |     17929     |      21250       | 0.3729  | 0.1974  | 0.3578  |   0.3576   |   0.0218    | 14.1014 |        13.2441        | 0.3647 |
+|    1.4522     | 13.99 | 509  |     1.3953      |   8047   |   3130   |   1564   |   811    |  16789   |  14585   |  12381   |  10177   |   47.9302    |   21.4604    |   12.6323    |    7.9689    |     0.7667      |     16789     |      21250       | 0.3712  |  0.197  | 0.3582  |   0.3581   |   0.0227    | 13.7526 |        12.515         | 0.3627 |
+|     1.407     | 14.98 | 545  |     1.3759      |   8498   |   3358   |   1703   |   877    |  17923   |  15719   |  13515   |  11311   |   47.4139    |   21.3627    |   12.6008    |    7.7535    |     0.8306      |     17923     |      21250       | 0.3856  | 0.2063  | 0.3709  |   0.3706   |   0.0213    | 14.7315 |        13.2849        | 0.3772 |
+|    1.3294     | 15.99 | 582  |     1.3776      |   8481   |   3407   |   1721   |   883    |  17451   |  15247   |  13043   |  10839   |   48.5989    |   22.3454    |   13.1948    |    8.1465    |     0.8044      |     17451     |      21250       | 0.3907  |  0.211  | 0.3766  |   0.3766   |    0.024    | 14.868  |        12.9142        | 0.3822 |
+|    1.3294     | 16.98 | 618  |     1.3803      |   8633   |   3464   |   1767   |   923    |  18004   |  15800   |  13596   |  11392   |   47.9505    |   21.9241    |   12.9965    |    8.1022    |      0.835      |     18004     |      21250       | 0.3946  | 0.2133  | 0.3801  |   0.3798   |   0.0263    | 15.2312 |        13.3103        | 0.3868 |
+|    1.2605     | 18.0  | 655  |     1.3710      |   8560   |   3376   |   1695   |   880    |  17830   |  15626   |  13422   |  11218   |    48.009    |    21.605    |   12.6285    |    7.8445    |     0.8255      |     17830     |      21250       | 0.3922  | 0.2092  | 0.3778  |   0.3775   |   0.0231    | 14.779  |        13.1665        | 0.3846 |
+|    1.2667     | 18.99 | 691  |     1.3694      |   8664   |   3455   |   1733   |   882    |  17834   |  15630   |  13426   |  11222   |   48.5814    |   22.1049    |   12.9078    |    7.8596    |     0.8257      |     17834     |      21250       | 0.3987  | 0.2138  | 0.3853  |   0.3851   |   0.0227    | 15.0008 |        13.2232        | 0.3906 |
+|    1.2074     | 19.79 | 720  |     1.3658      |   8770   |   3465   |   1737   |   880    |  18039   |  15835   |  13631   |  11427   |   48.6169    |   21.8819    |    12.743    |    7.7011    |     0.8369      |     18039     |      21250       | 0.4025  |  0.215  | 0.3883  |   0.3879   |   0.0227    | 15.0442 |        13.4424        | 0.3941 |
+
+
+### Framework versions
+
+- Transformers 4.32.1
+- Pytorch 2.1.0
+- Datasets 2.12.0
+- Tokenizers 0.13.3

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<hl>": 250100
+}

all_results.json

@@ -0,0 +1,64 @@
+{
+    "epoch": 19.79,
+    "eval_bleu": 15.0442,
+    "eval_bp": 0.8369,
+    "eval_counts_1": 8770,
+    "eval_counts_2": 3465,
+    "eval_counts_3": 1737,
+    "eval_counts_4": 880,
+    "eval_exact_match": 0.0227,
+    "eval_f1": 0.3941,
+    "eval_gen_len": 13.4424,
+    "eval_loss": 1.365785837173462,
+    "eval_precisions_1": 48.6169,
+    "eval_precisions_2": 21.8819,
+    "eval_precisions_3": 12.743,
+    "eval_precisions_4": 7.7011,
+    "eval_ref_len": 21250,
+    "eval_rouge1": 0.4025,
+    "eval_rouge2": 0.215,
+    "eval_rougeL": 0.3883,
+    "eval_rougeLsum": 0.3879,
+    "eval_runtime": 459.1599,
+    "eval_samples": 2204,
+    "eval_samples_per_second": 4.8,
+    "eval_steps_per_second": 1.2,
+    "eval_sys_len": 18039,
+    "eval_totals_1": 18039,
+    "eval_totals_2": 15835,
+    "eval_totals_3": 13631,
+    "eval_totals_4": 11427,
+    "predict_bleu": 13.4665,
+    "predict_bp": 0.8682,
+    "predict_counts_1": 8267,
+    "predict_counts_2": 3066,
+    "predict_counts_3": 1475,
+    "predict_counts_4": 724,
+    "predict_exact_match": 0.0281,
+    "predict_f1": 0.37,
+    "predict_gen_len": 13.7772,
+    "predict_loss": 1.5614243745803833,
+    "predict_precisions_1": 45.3757,
+    "predict_precisions_2": 19.1446,
+    "predict_precisions_3": 10.6799,
+    "predict_precisions_4": 6.2376,
+    "predict_ref_len": 20793,
+    "predict_rouge1": 0.3804,
+    "predict_rouge2": 0.1994,
+    "predict_rougeL": 0.3664,
+    "predict_rougeLsum": 0.3665,
+    "predict_runtime": 461.6889,
+    "predict_samples": 2204,
+    "predict_samples_per_second": 4.774,
+    "predict_steps_per_second": 1.193,
+    "predict_sys_len": 18219,
+    "predict_totals_1": 18219,
+    "predict_totals_2": 16015,
+    "predict_totals_3": 13811,
+    "predict_totals_4": 11607,
+    "train_loss": 2.0875697082943385,
+    "train_runtime": 23544.6757,
+    "train_samples": 9314,
+    "train_samples_per_second": 7.912,
+    "train_steps_per_second": 0.031
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "google/mt5-base",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "model_type": "mt5",
+  "num_beams": 4,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.32.1",
+  "use_cache": true,
+  "vocab_size": 250112
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "num_beams": 4,
+  "pad_token_id": 0,
+  "transformers_version": "4.32.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0154c470a501e9cd0362780d5a5221550c7a9cd47d8cd39683a81edb84cfde9
+size 2329638768

special_tokens_map.json

@@ -0,0 +1,8 @@
+{
+  "additional_special_tokens": [
+    "<hl>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c58c3dc929366af7c460d31895a225edc5077f5fb4438735a9896a78ab9842d7
+size 16330813

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "additional_special_tokens": null,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,768 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 19.78531558608845,
+  "eval_steps": 500,
+  "global_step": 720,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.99,
+      "learning_rate": 0.0001,
+      "loss": 8.9608,
+      "step": 36
+    },
+    {
+      "epoch": 0.99,
+      "eval_bleu": 0.2352,
+      "eval_bp": 0.828,
+      "eval_counts_1": 2306,
+      "eval_counts_2": 50,
+      "eval_counts_3": 12,
+      "eval_counts_4": 2,
+      "eval_exact_match": 0.0,
+      "eval_f1": 0.0092,
+      "eval_gen_len": 3.1969,
+      "eval_loss": 2.8882896900177,
+      "eval_precisions_1": 12.9,
+      "eval_precisions_2": 0.319,
+      "eval_precisions_3": 0.0891,
+      "eval_precisions_4": 0.0178,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.0081,
+      "eval_rouge2": 0.0022,
+      "eval_rougeL": 0.0078,
+      "eval_rougeLsum": 0.0078,
+      "eval_runtime": 386.3015,
+      "eval_samples_per_second": 5.705,
+      "eval_steps_per_second": 1.426,
+      "eval_sys_len": 17876,
+      "eval_totals_1": 17876,
+      "eval_totals_2": 15672,
+      "eval_totals_3": 13468,
+      "eval_totals_4": 11264,
+      "step": 36
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 0.0001,
+      "loss": 3.2364,
+      "step": 72
+    },
+    {
+      "epoch": 1.98,
+      "eval_bleu": 6.7083,
+      "eval_bp": 0.9954,
+      "eval_counts_1": 6125,
+      "eval_counts_2": 1727,
+      "eval_counts_3": 687,
+      "eval_counts_4": 277,
+      "eval_exact_match": 0.0018,
+      "eval_f1": 0.2514,
+      "eval_gen_len": 11.8072,
+      "eval_loss": 1.9241770505905151,
+      "eval_precisions_1": 28.9571,
+      "eval_precisions_2": 9.1144,
+      "eval_precisions_3": 4.103,
+      "eval_precisions_4": 1.9051,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.2457,
+      "eval_rouge2": 0.1026,
+      "eval_rougeL": 0.2345,
+      "eval_rougeLsum": 0.2346,
+      "eval_runtime": 440.0537,
+      "eval_samples_per_second": 5.008,
+      "eval_steps_per_second": 1.252,
+      "eval_sys_len": 21152,
+      "eval_totals_1": 21152,
+      "eval_totals_2": 18948,
+      "eval_totals_3": 16744,
+      "eval_totals_4": 14540,
+      "step": 72
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 0.0001,
+      "loss": 2.4963,
+      "step": 109
+    },
+    {
+      "epoch": 3.0,
+      "eval_bleu": 9.1493,
+      "eval_bp": 0.752,
+      "eval_counts_1": 6903,
+      "eval_counts_2": 2271,
+      "eval_counts_3": 975,
+      "eval_counts_4": 409,
+      "eval_exact_match": 0.01,
+      "eval_f1": 0.2909,
+      "eval_gen_len": 12.176,
+      "eval_loss": 1.6558014154434204,
+      "eval_precisions_1": 41.7428,
+      "eval_precisions_2": 15.8446,
+      "eval_precisions_3": 8.0386,
+      "eval_precisions_4": 4.1209,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.2966,
+      "eval_rouge2": 0.1415,
+      "eval_rougeL": 0.2854,
+      "eval_rougeLsum": 0.2852,
+      "eval_runtime": 434.1741,
+      "eval_samples_per_second": 5.076,
+      "eval_steps_per_second": 1.269,
+      "eval_sys_len": 16537,
+      "eval_totals_1": 16537,
+      "eval_totals_2": 14333,
+      "eval_totals_3": 12129,
+      "eval_totals_4": 9925,
+      "step": 109
+    },
+    {
+      "epoch": 3.98,
+      "learning_rate": 0.0001,
+      "loss": 2.2314,
+      "step": 145
+    },
+    {
+      "epoch": 3.98,
+      "eval_bleu": 10.187,
+      "eval_bp": 0.7573,
+      "eval_counts_1": 7160,
+      "eval_counts_2": 2440,
+      "eval_counts_3": 1098,
+      "eval_counts_4": 501,
+      "eval_exact_match": 0.0136,
+      "eval_f1": 0.3069,
+      "eval_gen_len": 12.157,
+      "eval_loss": 1.5771422386169434,
+      "eval_precisions_1": 43.0625,
+      "eval_precisions_2": 16.9174,
+      "eval_precisions_3": 8.986,
+      "eval_precisions_4": 5.0025,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.314,
+      "eval_rouge2": 0.1535,
+      "eval_rougeL": 0.3028,
+      "eval_rougeLsum": 0.3028,
+      "eval_runtime": 436.5308,
+      "eval_samples_per_second": 5.049,
+      "eval_steps_per_second": 1.262,
+      "eval_sys_len": 16627,
+      "eval_totals_1": 16627,
+      "eval_totals_2": 14423,
+      "eval_totals_3": 12219,
+      "eval_totals_4": 10015,
+      "step": 145
+    },
+    {
+      "epoch": 4.97,
+      "learning_rate": 0.0001,
+      "loss": 2.0578,
+      "step": 181
+    },
+    {
+      "epoch": 4.97,
+      "eval_bleu": 11.0621,
+      "eval_bp": 0.7961,
+      "eval_counts_1": 7447,
+      "eval_counts_2": 2625,
+      "eval_counts_3": 1214,
+      "eval_counts_4": 566,
+      "eval_exact_match": 0.0163,
+      "eval_f1": 0.32,
+      "eval_gen_len": 12.5585,
+      "eval_loss": 1.5346813201904297,
+      "eval_precisions_1": 43.0338,
+      "eval_precisions_2": 17.383,
+      "eval_precisions_3": 9.413,
+      "eval_precisions_4": 5.2932,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3286,
+      "eval_rouge2": 0.1628,
+      "eval_rougeL": 0.3146,
+      "eval_rougeLsum": 0.3146,
+      "eval_runtime": 444.2911,
+      "eval_samples_per_second": 4.961,
+      "eval_steps_per_second": 1.24,
+      "eval_sys_len": 17305,
+      "eval_totals_1": 17305,
+      "eval_totals_2": 15101,
+      "eval_totals_3": 12897,
+      "eval_totals_4": 10693,
+      "step": 181
+    },
+    {
+      "epoch": 5.99,
+      "learning_rate": 0.0001,
+      "loss": 1.8928,
+      "step": 218
+    },
+    {
+      "epoch": 5.99,
+      "eval_bleu": 11.4063,
+      "eval_bp": 0.7556,
+      "eval_counts_1": 7396,
+      "eval_counts_2": 2659,
+      "eval_counts_3": 1257,
+      "eval_counts_4": 611,
+      "eval_exact_match": 0.0177,
+      "eval_f1": 0.3234,
+      "eval_gen_len": 12.1692,
+      "eval_loss": 1.512817144393921,
+      "eval_precisions_1": 44.5596,
+      "eval_precisions_2": 18.473,
+      "eval_precisions_3": 10.3117,
+      "eval_precisions_4": 6.1186,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3326,
+      "eval_rouge2": 0.1684,
+      "eval_rougeL": 0.3198,
+      "eval_rougeLsum": 0.3198,
+      "eval_runtime": 441.07,
+      "eval_samples_per_second": 4.997,
+      "eval_steps_per_second": 1.249,
+      "eval_sys_len": 16598,
+      "eval_totals_1": 16598,
+      "eval_totals_2": 14394,
+      "eval_totals_3": 12190,
+      "eval_totals_4": 9986,
+      "step": 218
+    },
+    {
+      "epoch": 6.98,
+      "learning_rate": 0.0001,
+      "loss": 1.8573,
+      "step": 254
+    },
+    {
+      "epoch": 6.98,
+      "eval_bleu": 11.8292,
+      "eval_bp": 0.7631,
+      "eval_counts_1": 7531,
+      "eval_counts_2": 2758,
+      "eval_counts_3": 1313,
+      "eval_counts_4": 641,
+      "eval_exact_match": 0.0163,
+      "eval_f1": 0.327,
+      "eval_gen_len": 12.3035,
+      "eval_loss": 1.4735780954360962,
+      "eval_precisions_1": 45.0203,
+      "eval_precisions_2": 18.9893,
+      "eval_precisions_3": 10.6575,
+      "eval_precisions_4": 6.3365,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3349,
+      "eval_rouge2": 0.1717,
+      "eval_rougeL": 0.3216,
+      "eval_rougeLsum": 0.3216,
+      "eval_runtime": 442.6304,
+      "eval_samples_per_second": 4.979,
+      "eval_steps_per_second": 1.245,
+      "eval_sys_len": 16728,
+      "eval_totals_1": 16728,
+      "eval_totals_2": 14524,
+      "eval_totals_3": 12320,
+      "eval_totals_4": 10116,
+      "step": 254
+    },
+    {
+      "epoch": 8.0,
+      "learning_rate": 0.0001,
+      "loss": 1.7361,
+      "step": 291
+    },
+    {
+      "epoch": 8.0,
+      "eval_bleu": 12.2208,
+      "eval_bp": 0.7747,
+      "eval_counts_1": 7658,
+      "eval_counts_2": 2849,
+      "eval_counts_3": 1368,
+      "eval_counts_4": 668,
+      "eval_exact_match": 0.0181,
+      "eval_f1": 0.3334,
+      "eval_gen_len": 12.4628,
+      "eval_loss": 1.4544174671173096,
+      "eval_precisions_1": 45.2387,
+      "eval_precisions_2": 19.3494,
+      "eval_precisions_3": 10.9265,
+      "eval_precisions_4": 6.4754,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3414,
+      "eval_rouge2": 0.1762,
+      "eval_rougeL": 0.3283,
+      "eval_rougeLsum": 0.3284,
+      "eval_runtime": 442.3648,
+      "eval_samples_per_second": 4.982,
+      "eval_steps_per_second": 1.246,
+      "eval_sys_len": 16928,
+      "eval_totals_1": 16928,
+      "eval_totals_2": 14724,
+      "eval_totals_3": 12520,
+      "eval_totals_4": 10316,
+      "step": 291
+    },
+    {
+      "epoch": 8.99,
+      "learning_rate": 0.0001,
+      "loss": 1.7162,
+      "step": 327
+    },
+    {
+      "epoch": 8.99,
+      "eval_bleu": 12.4536,
+      "eval_bp": 0.767,
+      "eval_counts_1": 7703,
+      "eval_counts_2": 2891,
+      "eval_counts_3": 1390,
+      "eval_counts_4": 694,
+      "eval_exact_match": 0.0159,
+      "eval_f1": 0.3374,
+      "eval_gen_len": 12.4174,
+      "eval_loss": 1.4459445476531982,
+      "eval_precisions_1": 45.8648,
+      "eval_precisions_2": 19.8136,
+      "eval_precisions_3": 11.2214,
+      "eval_precisions_4": 6.8153,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3454,
+      "eval_rouge2": 0.1785,
+      "eval_rougeL": 0.3325,
+      "eval_rougeLsum": 0.3323,
+      "eval_runtime": 436.4836,
+      "eval_samples_per_second": 5.049,
+      "eval_steps_per_second": 1.262,
+      "eval_sys_len": 16795,
+      "eval_totals_1": 16795,
+      "eval_totals_2": 14591,
+      "eval_totals_3": 12387,
+      "eval_totals_4": 10183,
+      "step": 327
+    },
+    {
+      "epoch": 9.98,
+      "learning_rate": 0.0001,
+      "loss": 1.6589,
+      "step": 363
+    },
+    {
+      "epoch": 9.98,
+      "eval_bleu": 12.8553,
+      "eval_bp": 0.8002,
+      "eval_counts_1": 7889,
+      "eval_counts_2": 2983,
+      "eval_counts_3": 1449,
+      "eval_counts_4": 719,
+      "eval_exact_match": 0.0172,
+      "eval_f1": 0.3435,
+      "eval_gen_len": 12.7101,
+      "eval_loss": 1.438312292098999,
+      "eval_precisions_1": 45.4017,
+      "eval_precisions_2": 19.6612,
+      "eval_precisions_3": 11.1737,
+      "eval_precisions_4": 6.6797,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3519,
+      "eval_rouge2": 0.1816,
+      "eval_rougeL": 0.3375,
+      "eval_rougeLsum": 0.3372,
+      "eval_runtime": 449.3427,
+      "eval_samples_per_second": 4.905,
+      "eval_steps_per_second": 1.226,
+      "eval_sys_len": 17376,
+      "eval_totals_1": 17376,
+      "eval_totals_2": 15172,
+      "eval_totals_3": 12968,
+      "eval_totals_4": 10764,
+      "step": 363
+    },
+    {
+      "epoch": 10.99,
+      "learning_rate": 0.0001,
+      "loss": 1.5571,
+      "step": 400
+    },
+    {
+      "epoch": 10.99,
+      "eval_bleu": 12.9671,
+      "eval_bp": 0.7894,
+      "eval_counts_1": 7889,
+      "eval_counts_2": 2994,
+      "eval_counts_3": 1457,
+      "eval_counts_4": 736,
+      "eval_exact_match": 0.02,
+      "eval_f1": 0.3457,
+      "eval_gen_len": 12.6466,
+      "eval_loss": 1.4213731288909912,
+      "eval_precisions_1": 45.9063,
+      "eval_precisions_2": 19.9853,
+      "eval_precisions_3": 11.4033,
+      "eval_precisions_4": 6.9611,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3529,
+      "eval_rouge2": 0.1845,
+      "eval_rougeL": 0.3392,
+      "eval_rougeLsum": 0.3393,
+      "eval_runtime": 440.5687,
+      "eval_samples_per_second": 5.003,
+      "eval_steps_per_second": 1.251,
+      "eval_sys_len": 17185,
+      "eval_totals_1": 17185,
+      "eval_totals_2": 14981,
+      "eval_totals_3": 12777,
+      "eval_totals_4": 10573,
+      "step": 400
+    },
+    {
+      "epoch": 11.98,
+      "learning_rate": 0.0001,
+      "loss": 1.5502,
+      "step": 436
+    },
+    {
+      "epoch": 11.98,
+      "eval_bleu": 13.0741,
+      "eval_bp": 0.7712,
+      "eval_counts_1": 7930,
+      "eval_counts_2": 3008,
+      "eval_counts_3": 1477,
+      "eval_counts_4": 741,
+      "eval_exact_match": 0.0213,
+      "eval_f1": 0.3541,
+      "eval_gen_len": 12.4483,
+      "eval_loss": 1.4135174751281738,
+      "eval_precisions_1": 47.0121,
+      "eval_precisions_2": 20.5128,
+      "eval_precisions_3": 11.8539,
+      "eval_precisions_4": 7.225,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3619,
+      "eval_rouge2": 0.189,
+      "eval_rougeL": 0.3492,
+      "eval_rougeLsum": 0.3491,
+      "eval_runtime": 443.1145,
+      "eval_samples_per_second": 4.974,
+      "eval_steps_per_second": 1.243,
+      "eval_sys_len": 16868,
+      "eval_totals_1": 16868,
+      "eval_totals_2": 14664,
+      "eval_totals_3": 12460,
+      "eval_totals_4": 10256,
+      "step": 436
+    },
+    {
+      "epoch": 13.0,
+      "learning_rate": 0.0001,
+      "loss": 1.4564,
+      "step": 473
+    },
+    {
+      "epoch": 13.0,
+      "eval_bleu": 14.1014,
+      "eval_bp": 0.8309,
+      "eval_counts_1": 8268,
+      "eval_counts_2": 3200,
+      "eval_counts_3": 1616,
+      "eval_counts_4": 837,
+      "eval_exact_match": 0.0218,
+      "eval_f1": 0.3647,
+      "eval_gen_len": 13.2441,
+      "eval_loss": 1.3942722082138062,
+      "eval_precisions_1": 46.1152,
+      "eval_precisions_2": 20.3498,
+      "eval_precisions_3": 11.9518,
+      "eval_precisions_4": 7.396,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3729,
+      "eval_rouge2": 0.1974,
+      "eval_rougeL": 0.3578,
+      "eval_rougeLsum": 0.3576,
+      "eval_runtime": 460.2282,
+      "eval_samples_per_second": 4.789,
+      "eval_steps_per_second": 1.197,
+      "eval_sys_len": 17929,
+      "eval_totals_1": 17929,
+      "eval_totals_2": 15725,
+      "eval_totals_3": 13521,
+      "eval_totals_4": 11317,
+      "step": 473
+    },
+    {
+      "epoch": 13.99,
+      "learning_rate": 0.0001,
+      "loss": 1.4522,
+      "step": 509
+    },
+    {
+      "epoch": 13.99,
+      "eval_bleu": 13.7526,
+      "eval_bp": 0.7667,
+      "eval_counts_1": 8047,
+      "eval_counts_2": 3130,
+      "eval_counts_3": 1564,
+      "eval_counts_4": 811,
+      "eval_exact_match": 0.0227,
+      "eval_f1": 0.3627,
+      "eval_gen_len": 12.515,
+      "eval_loss": 1.3952871561050415,
+      "eval_precisions_1": 47.9302,
+      "eval_precisions_2": 21.4604,
+      "eval_precisions_3": 12.6323,
+      "eval_precisions_4": 7.9689,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3712,
+      "eval_rouge2": 0.197,
+      "eval_rougeL": 0.3582,
+      "eval_rougeLsum": 0.3581,
+      "eval_runtime": 437.5396,
+      "eval_samples_per_second": 5.037,
+      "eval_steps_per_second": 1.259,
+      "eval_sys_len": 16789,
+      "eval_totals_1": 16789,
+      "eval_totals_2": 14585,
+      "eval_totals_3": 12381,
+      "eval_totals_4": 10177,
+      "step": 509
+    },
+    {
+      "epoch": 14.98,
+      "learning_rate": 0.0001,
+      "loss": 1.407,
+      "step": 545
+    },
+    {
+      "epoch": 14.98,
+      "eval_bleu": 14.7315,
+      "eval_bp": 0.8306,
+      "eval_counts_1": 8498,
+      "eval_counts_2": 3358,
+      "eval_counts_3": 1703,
+      "eval_counts_4": 877,
+      "eval_exact_match": 0.0213,
+      "eval_f1": 0.3772,
+      "eval_gen_len": 13.2849,
+      "eval_loss": 1.3759350776672363,
+      "eval_precisions_1": 47.4139,
+      "eval_precisions_2": 21.3627,
+      "eval_precisions_3": 12.6008,
+      "eval_precisions_4": 7.7535,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3856,
+      "eval_rouge2": 0.2063,
+      "eval_rougeL": 0.3709,
+      "eval_rougeLsum": 0.3706,
+      "eval_runtime": 453.6157,
+      "eval_samples_per_second": 4.859,
+      "eval_steps_per_second": 1.215,
+      "eval_sys_len": 17923,
+      "eval_totals_1": 17923,
+      "eval_totals_2": 15719,
+      "eval_totals_3": 13515,
+      "eval_totals_4": 11311,
+      "step": 545
+    },
+    {
+      "epoch": 15.99,
+      "learning_rate": 0.0001,
+      "loss": 1.3294,
+      "step": 582
+    },
+    {
+      "epoch": 15.99,
+      "eval_bleu": 14.868,
+      "eval_bp": 0.8044,
+      "eval_counts_1": 8481,
+      "eval_counts_2": 3407,
+      "eval_counts_3": 1721,
+      "eval_counts_4": 883,
+      "eval_exact_match": 0.024,
+      "eval_f1": 0.3822,
+      "eval_gen_len": 12.9142,
+      "eval_loss": 1.3775662183761597,
+      "eval_precisions_1": 48.5989,
+      "eval_precisions_2": 22.3454,
+      "eval_precisions_3": 13.1948,
+      "eval_precisions_4": 8.1465,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3907,
+      "eval_rouge2": 0.211,
+      "eval_rougeL": 0.3766,
+      "eval_rougeLsum": 0.3766,
+      "eval_runtime": 448.6685,
+      "eval_samples_per_second": 4.912,
+      "eval_steps_per_second": 1.228,
+      "eval_sys_len": 17451,
+      "eval_totals_1": 17451,
+      "eval_totals_2": 15247,
+      "eval_totals_3": 13043,
+      "eval_totals_4": 10839,
+      "step": 582
+    },
+    {
+      "epoch": 16.98,
+      "learning_rate": 0.0001,
+      "loss": 1.3294,
+      "step": 618
+    },
+    {
+      "epoch": 16.98,
+      "eval_bleu": 15.2312,
+      "eval_bp": 0.835,
+      "eval_counts_1": 8633,
+      "eval_counts_2": 3464,
+      "eval_counts_3": 1767,
+      "eval_counts_4": 923,
+      "eval_exact_match": 0.0263,
+      "eval_f1": 0.3868,
+      "eval_gen_len": 13.3103,
+      "eval_loss": 1.380259394645691,
+      "eval_precisions_1": 47.9505,
+      "eval_precisions_2": 21.9241,
+      "eval_precisions_3": 12.9965,
+      "eval_precisions_4": 8.1022,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3946,
+      "eval_rouge2": 0.2133,
+      "eval_rougeL": 0.3801,
+      "eval_rougeLsum": 0.3798,
+      "eval_runtime": 456.612,
+      "eval_samples_per_second": 4.827,
+      "eval_steps_per_second": 1.207,
+      "eval_sys_len": 18004,
+      "eval_totals_1": 18004,
+      "eval_totals_2": 15800,
+      "eval_totals_3": 13596,
+      "eval_totals_4": 11392,
+      "step": 618
+    },
+    {
+      "epoch": 18.0,
+      "learning_rate": 0.0001,
+      "loss": 1.2605,
+      "step": 655
+    },
+    {
+      "epoch": 18.0,
+      "eval_bleu": 14.779,
+      "eval_bp": 0.8255,
+      "eval_counts_1": 8560,
+      "eval_counts_2": 3376,
+      "eval_counts_3": 1695,
+      "eval_counts_4": 880,
+      "eval_exact_match": 0.0231,
+      "eval_f1": 0.3846,
+      "eval_gen_len": 13.1665,
+      "eval_loss": 1.3709588050842285,
+      "eval_precisions_1": 48.009,
+      "eval_precisions_2": 21.605,
+      "eval_precisions_3": 12.6285,
+      "eval_precisions_4": 7.8445,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3922,
+      "eval_rouge2": 0.2092,
+      "eval_rougeL": 0.3778,
+      "eval_rougeLsum": 0.3775,
+      "eval_runtime": 456.164,
+      "eval_samples_per_second": 4.832,
+      "eval_steps_per_second": 1.208,
+      "eval_sys_len": 17830,
+      "eval_totals_1": 17830,
+      "eval_totals_2": 15626,
+      "eval_totals_3": 13422,
+      "eval_totals_4": 11218,
+      "step": 655
+    },
+    {
+      "epoch": 18.99,
+      "learning_rate": 0.0001,
+      "loss": 1.2667,
+      "step": 691
+    },
+    {
+      "epoch": 18.99,
+      "eval_bleu": 15.0008,
+      "eval_bp": 0.8257,
+      "eval_counts_1": 8664,
+      "eval_counts_2": 3455,
+      "eval_counts_3": 1733,
+      "eval_counts_4": 882,
+      "eval_exact_match": 0.0227,
+      "eval_f1": 0.3906,
+      "eval_gen_len": 13.2232,
+      "eval_loss": 1.3694192171096802,
+      "eval_precisions_1": 48.5814,
+      "eval_precisions_2": 22.1049,
+      "eval_precisions_3": 12.9078,
+      "eval_precisions_4": 7.8596,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3987,
+      "eval_rouge2": 0.2138,
+      "eval_rougeL": 0.3853,
+      "eval_rougeLsum": 0.3851,
+      "eval_runtime": 454.2362,
+      "eval_samples_per_second": 4.852,
+      "eval_steps_per_second": 1.213,
+      "eval_sys_len": 17834,
+      "eval_totals_1": 17834,
+      "eval_totals_2": 15630,
+      "eval_totals_3": 13426,
+      "eval_totals_4": 11222,
+      "step": 691
+    },
+    {
+      "epoch": 19.79,
+      "learning_rate": 0.0001,
+      "loss": 1.2074,
+      "step": 720
+    },
+    {
+      "epoch": 19.79,
+      "eval_bleu": 15.0442,
+      "eval_bp": 0.8369,
+      "eval_counts_1": 8770,
+      "eval_counts_2": 3465,
+      "eval_counts_3": 1737,
+      "eval_counts_4": 880,
+      "eval_exact_match": 0.0227,
+      "eval_f1": 0.3941,
+      "eval_gen_len": 13.4424,
+      "eval_loss": 1.365785837173462,
+      "eval_precisions_1": 48.6169,
+      "eval_precisions_2": 21.8819,
+      "eval_precisions_3": 12.743,
+      "eval_precisions_4": 7.7011,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4025,
+      "eval_rouge2": 0.215,
+      "eval_rougeL": 0.3883,
+      "eval_rougeLsum": 0.3879,
+      "eval_runtime": 459.1457,
+      "eval_samples_per_second": 4.8,
+      "eval_steps_per_second": 1.2,
+      "eval_sys_len": 18039,
+      "eval_totals_1": 18039,
+      "eval_totals_2": 15835,
+      "eval_totals_3": 13631,
+      "eval_totals_4": 11427,
+      "step": 720
+    },
+    {
+      "epoch": 19.79,
+      "step": 720,
+      "total_flos": 4.419252384883016e+17,
+      "train_loss": 2.0875697082943385,
+      "train_runtime": 23544.6757,
+      "train_samples_per_second": 7.912,
+      "train_steps_per_second": 0.031
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 720,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "total_flos": 4.419252384883016e+17,
+  "trial_name": null,
+  "trial_params": null
+}