Initial commit

.gitattributes

@@ -0,0 +1,4 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+runs/** filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,128 @@
+---
+language:
+  - de
+tags:
+  - question-generation
+  - german
+  - text2text-generation
+  - generated_from_trainer
+datasets:
+  - lmqg/qg_dequad
+metrics:
+  - bleu4
+  - f1
+  - rouge
+  - exact_match
+model-index:
+  - name: german-jeopardy-mt5-large-128
+    results:
+      - task:
+          name: Sequence-to-sequence Language Modeling
+          type: text2text-generation
+        dataset:
+          name: lmqg/qg_dequad
+          type: default
+          args: default
+        metrics:
+          - name: BLEU-4
+            type: bleu4
+            value: 16.06
+          - name: F1
+            type: f1
+            value: 42.29
+          - name: ROUGE-1
+            type: rouge1
+            value: 43.40
+          - name: ROUGE-2
+            type: rouge2
+            value: 23.68
+          - name: ROUGE-L
+            type: rougel
+            value: 41.78
+          - name: ROUGE-Lsum
+            type: rougelsum
+            value: 41.79
+          - name: Exact Match
+            type: exact_match
+            value: 3.18
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# german-jeopardy-mt5-large-128
+
+This model is a fine-tuned version of [google/mt5-large](https://huggingface.co/google/mt5-large) on the [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad) dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.5487
+- Brevity Penalty: 0.9115
+- System Length: 19029
+- Reference Length: 20793
+- ROUGE-1: 43.40
+- ROUGE-2: 23.68
+- ROUGE-L: 41.78
+- ROUGE-Lsum: 41.79
+- Exact Match: 3.18
+- BLEU: 16.06
+- F1: 42.29
+
+## Model description
+
+See [google/mt5-large](https://huggingface.co/google/mt5-large) for the model architecture.  
+The model was trained on a single NVIDIA RTX 3090 GPU with 24GB of VRAM.
+
+## Intended uses & limitations
+
+This model can be used for question generation on German text.
+
+## Training and evaluation data
+
+See [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad).
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 7
+- gradient_accumulation_steps: 128
+- total_train_batch_size: 128
+- optimizer: Adafactor
+- lr_scheduler_type: constant
+- num_epochs: 20
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Counts 1 | Counts 2 | Counts 3 | Counts 4 | Totals 1 | Totals 2 | Totals 3 | Totals 4 | Precisions 1 | Precisions 2 | Precisions 3 | Precisions 4 | Brevity Penalty | System Length | Reference Length | ROUGE-1 | ROUGE-2 | ROUGE-L | ROUGE-Lsum | Exact Match |  BLEU   | Mean Generated Length |   F1   |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:------------:|:------------:|:------------:|:------------:|:---------------:|:-------------:|:----------------:|:-------:|:-------:|:-------:|:----------:|:-----------:|:-------:|:---------------------:|:------:|
+|    3.9659     | 0.99  |  72  |     1.4145      |   7244   |   2547   |   1183   |   565    |  16296   |  14092   |  11888   |   9684   |   44.4526    |   18.0741    |    9.9512    |    5.8344    |     0.7379      |     16296     |      21250       | 0.3213  | 0.1608  | 0.3091  |   0.309    |   0.0136    | 10.8438 |        11.7786        | 0.3139 |
+|    1.7081     | 1.99  | 145  |     1.2632      |   7865   |   3037   |   1498   |   759    |  16841   |  14637   |  12433   |  10229   |   46.7015    |   20.7488    |   12.0486    |    7.4201    |     0.7697      |     16841     |      21250       | 0.3577  |  0.189  | 0.3438  |   0.3439   |   0.0181    | 13.2044 |        12.225         | 0.3481 |
+|    1.4856     |  3.0  | 218  |     1.1974      |   8608   |   3519   |   1818   |   969    |  17627   |  15423   |  13219   |  11015   |   48.8342    |   22.8166    |   13.7529    |    8.7971    |     0.8142      |     17627     |      21250       | 0.3969  | 0.2181  |  0.381  |   0.3812   |   0.0268    | 15.6014 |        13.0027        | 0.3882 |
+|    1.3277     |  4.0  | 291  |     1.1394      |   9018   |   3702   |   1907   |   1029   |  17465   |  15261   |  13057   |  10853   |   51.6347    |   24.2579    |   14.6052    |    9.4812    |     0.8052      |     17465     |      21250       |  0.424  | 0.2321  | 0.4087  |   0.4085   |   0.0313    | 16.4313 |        12.8716        | 0.4156 |
+|    1.2314     | 4.99  | 363  |     1.1193      |   9240   |   3869   |   1994   |   1076   |  17794   |  15590   |  13386   |  11182   |   51.9276    |   24.8172    |   14.8962    |    9.6226    |     0.8235      |     17794     |      21250       | 0.4336  | 0.2413  | 0.4183  |   0.418    |   0.0363    | 17.0718 |        13.2137        | 0.4256 |
+|    1.1264     | 5.99  | 436  |     1.1086      |   9263   |   3908   |   2055   |   1127   |  17502   |  15298   |  13094   |  10890   |   52.9254    |   25.5458    |   15.6942    |   10.3489    |     0.8072      |     17502     |      21250       | 0.4383  | 0.2452  | 0.4239  |   0.4237   |   0.0372    | 17.4744 |        13.034         | 0.4309 |
+|    1.0469     |  7.0  | 509  |     1.1038      |   9434   |   4034   |   2146   |   1189   |  18028   |  15824   |  13620   |  11416   |   52.3297    |   25.4929    |   15.7562    |   10.4152    |     0.8363      |     18028     |      21250       | 0.4433  | 0.2505  | 0.4286  |   0.4282   |    0.039    | 18.0906 |        13.422         | 0.4348 |
+|    0.9874     |  8.0  | 582  |     1.0990      |   9746   |   4265   |   2287   |   1285   |  18351   |  16147   |  13943   |  11739   |   53.1088    |   26.4136    |   16.4025    |   10.9464    |     0.8539      |     18351     |      21250       |  0.457  | 0.2627  | 0.4417  |   0.4416   |   0.0454    | 19.1287 |        13.6466        | 0.4498 |
+|    0.9488     | 8.99  | 654  |     1.1175      |   9484   |   4062   |   2158   |   1197   |  17831   |  15627   |  13423   |  11219   |   53.1883    |   25.9935    |   16.0769    |   10.6694    |     0.8255      |     17831     |      21250       | 0.4482  | 0.2548  | 0.4338  |   0.4333   |   0.0431    | 18.2172 |        13.2763        | 0.4399 |
+|    0.8893     | 9.99  | 727  |     1.1222      |   9650   |   4205   |   2289   |   1289   |  18017   |  15813   |  13609   |  11405   |   53.5605    |    26.592    |   16.8198    |   11.3021    |     0.8357      |     18017     |      21250       | 0.4543  |  0.262  | 0.4396  |   0.4394   |   0.0463    | 19.064  |        13.4251        | 0.4472 |
+|    0.8362     | 10.99 | 800  |     1.1342      |   9706   |   4232   |   2279   |   1281   |  18232   |  16028   |  13824   |  11620   |   53.2361    |   26.4038    |   16.4858    |   11.0241    |     0.8474      |     18232     |      21250       | 0.4551  | 0.2632  | 0.4395  |   0.4393   |   0.0472    | 19.052  |        13.6021        | 0.4473 |
+|    0.7835     | 12.0  | 873  |     1.1427      |   9802   |   4280   |   2292   |   1285   |  18491   |  16287   |  14083   |  11879   |   53.0096    |   26.2786    |   16.2749    |   10.8174    |     0.8614      |     18491     |      21250       |  0.458  | 0.2634  | 0.4414  |   0.4412   |   0.0472    | 19.169  |        14.0168        | 0.4497 |
+|    0.7441     | 12.99 | 945  |     1.1669      |   9816   |   4323   |   2334   |   1294   |  18498   |  16294   |  14090   |  11886   |   53.0652    |   26.5312    |   16.5649    |   10.8868    |     0.8618      |     18498     |      21250       | 0.4577  | 0.2659  | 0.4418  |   0.4417   |   0.0463    | 19.3443 |        13.8348        | 0.4493 |
+|    0.7012     | 13.99 | 1018 |     1.1740      |   9856   |   4364   |   2375   |   1360   |  18537   |  16333   |  14129   |  11925   |   53.1693    |   26.7189    |   16.8094    |   11.4046    |     0.8639      |     18537     |      21250       | 0.4591  | 0.2653  |  0.443  |   0.4428   |   0.0476    | 19.7341 |        13.976         | 0.4514 |
+|    0.6597     | 14.99 | 1091 |     1.1987      |   9780   |   4292   |   2336   |   1302   |  18468   |  16264   |  14060   |  11856   |   52.9565    |   26.3896    |   16.6145    |   10.9818    |     0.8602      |     18468     |      21250       |  0.457  | 0.2633  | 0.4418  |   0.4416   |   0.0485    | 19.3289 |        13.8802        | 0.4492 |
+|    0.6236     | 16.0  | 1164 |     1.2135      |   9931   |   4388   |   2390   |   1359   |  18717   |  16513   |  14309   |  12105   |   53.0587    |    26.573    |   16.7028    |   11.2268    |     0.8734      |     18717     |      21250       | 0.4618  | 0.2682  | 0.4452  |   0.445    |   0.0495    | 19.8055 |        14.044         | 0.4538 |
+|    0.5933     | 17.0  | 1237 |     1.2305      |   9806   |   4316   |   2366   |   1348   |  18566   |  16362   |  14158   |  11954   |    52.817    |   26.3782    |   16.7114    |   11.2766    |     0.8654      |     18566     |      21250       | 0.4571  | 0.2628  | 0.4407  |   0.4409   |    0.049    | 19.5893 |        14.0622        | 0.4485 |
+|    0.5622     | 17.99 | 1309 |     1.2796      |   9787   |   4306   |   2346   |   1338   |  18559   |  16355   |  14151   |  11947   |   52.7345    |   26.3283    |   16.5783    |   11.1995    |      0.865      |     18559     |      21250       | 0.4549  | 0.2609  | 0.4383  |   0.4382   |   0.0476    | 19.4914 |        13.7763        | 0.447  |
+|    0.5275     | 18.99 | 1382 |     1.2833      |   9918   |   4363   |   2374   |   1355   |  18950   |  16746   |  14542   |  12338   |   52.3377    |    26.054    |   16.3251    |   10.9823    |     0.8857      |     18950     |      21250       | 0.4573  | 0.2624  |  0.441  |   0.4408   |   0.0508    | 19.6947 |        14.1647        | 0.4499 |
+|    0.4986     | 19.79 | 1440 |     1.3059      |   9879   |   4315   |   2347   |   1324   |  18931   |  16727   |  14523   |  12319   |   52.1842    |   25.7966    |   16.1606    |   10.7476    |     0.8847      |     18931     |      21250       | 0.4564  | 0.2622  | 0.4407  |   0.4403   |   0.0495    | 19.4544 |        14.2827        | 0.4478 |
+
+
+### Framework versions
+
+- Transformers 4.32.1
+- Pytorch 2.1.0
+- Datasets 2.12.0
+- Tokenizers 0.13.3

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<hl>": 250100
+}

all_results.json

@@ -0,0 +1,64 @@
+{
+    "epoch": 19.79,
+    "eval_bleu": 19.4544,
+    "eval_bp": 0.8847,
+    "eval_counts_1": 9879,
+    "eval_counts_2": 4315,
+    "eval_counts_3": 2347,
+    "eval_counts_4": 1324,
+    "eval_exact_match": 0.0495,
+    "eval_f1": 0.4478,
+    "eval_gen_len": 14.2827,
+    "eval_loss": 1.3059108257293701,
+    "eval_precisions_1": 52.1842,
+    "eval_precisions_2": 25.7966,
+    "eval_precisions_3": 16.1606,
+    "eval_precisions_4": 10.7476,
+    "eval_ref_len": 21250,
+    "eval_rouge1": 0.4564,
+    "eval_rouge2": 0.2622,
+    "eval_rougeL": 0.4407,
+    "eval_rougeLsum": 0.4403,
+    "eval_runtime": 3710.7373,
+    "eval_samples": 2204,
+    "eval_samples_per_second": 0.594,
+    "eval_steps_per_second": 0.594,
+    "eval_sys_len": 18931,
+    "eval_totals_1": 18931,
+    "eval_totals_2": 16727,
+    "eval_totals_3": 14523,
+    "eval_totals_4": 12319,
+    "predict_bleu": 15.5923,
+    "predict_bp": 0.9152,
+    "predict_counts_1": 9130,
+    "predict_counts_2": 3573,
+    "predict_counts_3": 1732,
+    "predict_counts_4": 883,
+    "predict_exact_match": 0.0318,
+    "predict_f1": 0.4128,
+    "predict_gen_len": 14.554,
+    "predict_loss": 1.5979496240615845,
+    "predict_precisions_1": 47.801,
+    "predict_precisions_2": 21.147,
+    "predict_precisions_3": 11.7887,
+    "predict_precisions_4": 7.0708,
+    "predict_ref_len": 20793,
+    "predict_rouge1": 0.4233,
+    "predict_rouge2": 0.2296,
+    "predict_rougeL": 0.407,
+    "predict_rougeLsum": 0.4067,
+    "predict_runtime": 3256.8207,
+    "predict_samples": 2204,
+    "predict_samples_per_second": 0.677,
+    "predict_steps_per_second": 0.677,
+    "predict_sys_len": 19100,
+    "predict_totals_1": 19100,
+    "predict_totals_2": 16896,
+    "predict_totals_3": 14692,
+    "predict_totals_4": 12488,
+    "train_loss": 1.0667428798145717,
+    "train_runtime": 140813.6912,
+    "train_samples": 9314,
+    "train_samples_per_second": 1.323,
+    "train_steps_per_second": 0.01
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "google/mt5-large",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2816,
+  "d_kv": 64,
+  "d_model": 1024,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "model_type": "mt5",
+  "num_beams": 4,
+  "num_decoder_layers": 24,
+  "num_heads": 16,
+  "num_layers": 24,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.32.1",
+  "use_cache": true,
+  "vocab_size": 250112
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "num_beams": 4,
+  "pad_token_id": 0,
+  "transformers_version": "4.32.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d027a7e16cc798792ca2a7dd9102e4806e556b8d7c1fad130064107cab1a5bf5
+size 4918393736

special_tokens_map.json

@@ -0,0 +1,8 @@
+{
+  "additional_special_tokens": [
+    "<hl>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef4273f90fa3341fdb81c66eb7973b1651a8183e57c8ab9e9144aac61b48f9f
+size 16330550

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "additional_special_tokens": null,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,768 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 19.789564097058193,
+  "eval_steps": 500,
+  "global_step": 1440,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.99,
+      "learning_rate": 0.0001,
+      "loss": 3.9659,
+      "step": 72
+    },
+    {
+      "epoch": 0.99,
+      "eval_bleu": 10.8438,
+      "eval_bp": 0.7379,
+      "eval_counts_1": 7244,
+      "eval_counts_2": 2547,
+      "eval_counts_3": 1183,
+      "eval_counts_4": 565,
+      "eval_exact_match": 0.0136,
+      "eval_f1": 0.3139,
+      "eval_gen_len": 11.7786,
+      "eval_loss": 1.4144511222839355,
+      "eval_precisions_1": 44.4526,
+      "eval_precisions_2": 18.0741,
+      "eval_precisions_3": 9.9512,
+      "eval_precisions_4": 5.8344,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3213,
+      "eval_rouge2": 0.1608,
+      "eval_rougeL": 0.3091,
+      "eval_rougeLsum": 0.309,
+      "eval_runtime": 2106.9539,
+      "eval_samples_per_second": 1.046,
+      "eval_steps_per_second": 1.046,
+      "eval_sys_len": 16296,
+      "eval_totals_1": 16296,
+      "eval_totals_2": 14092,
+      "eval_totals_3": 11888,
+      "eval_totals_4": 9684,
+      "step": 72
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 0.0001,
+      "loss": 1.7081,
+      "step": 145
+    },
+    {
+      "epoch": 1.99,
+      "eval_bleu": 13.2044,
+      "eval_bp": 0.7697,
+      "eval_counts_1": 7865,
+      "eval_counts_2": 3037,
+      "eval_counts_3": 1498,
+      "eval_counts_4": 759,
+      "eval_exact_match": 0.0181,
+      "eval_f1": 0.3481,
+      "eval_gen_len": 12.225,
+      "eval_loss": 1.263157844543457,
+      "eval_precisions_1": 46.7015,
+      "eval_precisions_2": 20.7488,
+      "eval_precisions_3": 12.0486,
+      "eval_precisions_4": 7.4201,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3577,
+      "eval_rouge2": 0.189,
+      "eval_rougeL": 0.3438,
+      "eval_rougeLsum": 0.3439,
+      "eval_runtime": 3942.8178,
+      "eval_samples_per_second": 0.559,
+      "eval_steps_per_second": 0.559,
+      "eval_sys_len": 16841,
+      "eval_totals_1": 16841,
+      "eval_totals_2": 14637,
+      "eval_totals_3": 12433,
+      "eval_totals_4": 10229,
+      "step": 145
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 0.0001,
+      "loss": 1.4856,
+      "step": 218
+    },
+    {
+      "epoch": 3.0,
+      "eval_bleu": 15.6014,
+      "eval_bp": 0.8142,
+      "eval_counts_1": 8608,
+      "eval_counts_2": 3519,
+      "eval_counts_3": 1818,
+      "eval_counts_4": 969,
+      "eval_exact_match": 0.0268,
+      "eval_f1": 0.3882,
+      "eval_gen_len": 13.0027,
+      "eval_loss": 1.1974213123321533,
+      "eval_precisions_1": 48.8342,
+      "eval_precisions_2": 22.8166,
+      "eval_precisions_3": 13.7529,
+      "eval_precisions_4": 8.7971,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3969,
+      "eval_rouge2": 0.2181,
+      "eval_rougeL": 0.381,
+      "eval_rougeLsum": 0.3812,
+      "eval_runtime": 4069.754,
+      "eval_samples_per_second": 0.542,
+      "eval_steps_per_second": 0.542,
+      "eval_sys_len": 17627,
+      "eval_totals_1": 17627,
+      "eval_totals_2": 15423,
+      "eval_totals_3": 13219,
+      "eval_totals_4": 11015,
+      "step": 218
+    },
+    {
+      "epoch": 4.0,
+      "learning_rate": 0.0001,
+      "loss": 1.3277,
+      "step": 291
+    },
+    {
+      "epoch": 4.0,
+      "eval_bleu": 16.4313,
+      "eval_bp": 0.8052,
+      "eval_counts_1": 9018,
+      "eval_counts_2": 3702,
+      "eval_counts_3": 1907,
+      "eval_counts_4": 1029,
+      "eval_exact_match": 0.0313,
+      "eval_f1": 0.4156,
+      "eval_gen_len": 12.8716,
+      "eval_loss": 1.1393847465515137,
+      "eval_precisions_1": 51.6347,
+      "eval_precisions_2": 24.2579,
+      "eval_precisions_3": 14.6052,
+      "eval_precisions_4": 9.4812,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.424,
+      "eval_rouge2": 0.2321,
+      "eval_rougeL": 0.4087,
+      "eval_rougeLsum": 0.4085,
+      "eval_runtime": 4037.7601,
+      "eval_samples_per_second": 0.546,
+      "eval_steps_per_second": 0.546,
+      "eval_sys_len": 17465,
+      "eval_totals_1": 17465,
+      "eval_totals_2": 15261,
+      "eval_totals_3": 13057,
+      "eval_totals_4": 10853,
+      "step": 291
+    },
+    {
+      "epoch": 4.99,
+      "learning_rate": 0.0001,
+      "loss": 1.2314,
+      "step": 363
+    },
+    {
+      "epoch": 4.99,
+      "eval_bleu": 17.0718,
+      "eval_bp": 0.8235,
+      "eval_counts_1": 9240,
+      "eval_counts_2": 3869,
+      "eval_counts_3": 1994,
+      "eval_counts_4": 1076,
+      "eval_exact_match": 0.0363,
+      "eval_f1": 0.4256,
+      "eval_gen_len": 13.2137,
+      "eval_loss": 1.1193382740020752,
+      "eval_precisions_1": 51.9276,
+      "eval_precisions_2": 24.8172,
+      "eval_precisions_3": 14.8962,
+      "eval_precisions_4": 9.6226,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4336,
+      "eval_rouge2": 0.2413,
+      "eval_rougeL": 0.4183,
+      "eval_rougeLsum": 0.418,
+      "eval_runtime": 4116.6581,
+      "eval_samples_per_second": 0.535,
+      "eval_steps_per_second": 0.535,
+      "eval_sys_len": 17794,
+      "eval_totals_1": 17794,
+      "eval_totals_2": 15590,
+      "eval_totals_3": 13386,
+      "eval_totals_4": 11182,
+      "step": 363
+    },
+    {
+      "epoch": 5.99,
+      "learning_rate": 0.0001,
+      "loss": 1.1264,
+      "step": 436
+    },
+    {
+      "epoch": 5.99,
+      "eval_bleu": 17.4744,
+      "eval_bp": 0.8072,
+      "eval_counts_1": 9263,
+      "eval_counts_2": 3908,
+      "eval_counts_3": 2055,
+      "eval_counts_4": 1127,
+      "eval_exact_match": 0.0372,
+      "eval_f1": 0.4309,
+      "eval_gen_len": 13.034,
+      "eval_loss": 1.1085509061813354,
+      "eval_precisions_1": 52.9254,
+      "eval_precisions_2": 25.5458,
+      "eval_precisions_3": 15.6942,
+      "eval_precisions_4": 10.3489,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4383,
+      "eval_rouge2": 0.2452,
+      "eval_rougeL": 0.4239,
+      "eval_rougeLsum": 0.4237,
+      "eval_runtime": 3709.3886,
+      "eval_samples_per_second": 0.594,
+      "eval_steps_per_second": 0.594,
+      "eval_sys_len": 17502,
+      "eval_totals_1": 17502,
+      "eval_totals_2": 15298,
+      "eval_totals_3": 13094,
+      "eval_totals_4": 10890,
+      "step": 436
+    },
+    {
+      "epoch": 7.0,
+      "learning_rate": 0.0001,
+      "loss": 1.0469,
+      "step": 509
+    },
+    {
+      "epoch": 7.0,
+      "eval_bleu": 18.0906,
+      "eval_bp": 0.8363,
+      "eval_counts_1": 9434,
+      "eval_counts_2": 4034,
+      "eval_counts_3": 2146,
+      "eval_counts_4": 1189,
+      "eval_exact_match": 0.039,
+      "eval_f1": 0.4348,
+      "eval_gen_len": 13.422,
+      "eval_loss": 1.103752851486206,
+      "eval_precisions_1": 52.3297,
+      "eval_precisions_2": 25.4929,
+      "eval_precisions_3": 15.7562,
+      "eval_precisions_4": 10.4152,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4433,
+      "eval_rouge2": 0.2505,
+      "eval_rougeL": 0.4286,
+      "eval_rougeLsum": 0.4282,
+      "eval_runtime": 4081.2971,
+      "eval_samples_per_second": 0.54,
+      "eval_steps_per_second": 0.54,
+      "eval_sys_len": 18028,
+      "eval_totals_1": 18028,
+      "eval_totals_2": 15824,
+      "eval_totals_3": 13620,
+      "eval_totals_4": 11416,
+      "step": 509
+    },
+    {
+      "epoch": 8.0,
+      "learning_rate": 0.0001,
+      "loss": 0.9874,
+      "step": 582
+    },
+    {
+      "epoch": 8.0,
+      "eval_bleu": 19.1287,
+      "eval_bp": 0.8539,
+      "eval_counts_1": 9746,
+      "eval_counts_2": 4265,
+      "eval_counts_3": 2287,
+      "eval_counts_4": 1285,
+      "eval_exact_match": 0.0454,
+      "eval_f1": 0.4498,
+      "eval_gen_len": 13.6466,
+      "eval_loss": 1.0989724397659302,
+      "eval_precisions_1": 53.1088,
+      "eval_precisions_2": 26.4136,
+      "eval_precisions_3": 16.4025,
+      "eval_precisions_4": 10.9464,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.457,
+      "eval_rouge2": 0.2627,
+      "eval_rougeL": 0.4417,
+      "eval_rougeLsum": 0.4416,
+      "eval_runtime": 2875.9709,
+      "eval_samples_per_second": 0.766,
+      "eval_steps_per_second": 0.766,
+      "eval_sys_len": 18351,
+      "eval_totals_1": 18351,
+      "eval_totals_2": 16147,
+      "eval_totals_3": 13943,
+      "eval_totals_4": 11739,
+      "step": 582
+    },
+    {
+      "epoch": 8.99,
+      "learning_rate": 0.0001,
+      "loss": 0.9488,
+      "step": 654
+    },
+    {
+      "epoch": 8.99,
+      "eval_bleu": 18.2172,
+      "eval_bp": 0.8255,
+      "eval_counts_1": 9484,
+      "eval_counts_2": 4062,
+      "eval_counts_3": 2158,
+      "eval_counts_4": 1197,
+      "eval_exact_match": 0.0431,
+      "eval_f1": 0.4399,
+      "eval_gen_len": 13.2763,
+      "eval_loss": 1.1175453662872314,
+      "eval_precisions_1": 53.1883,
+      "eval_precisions_2": 25.9935,
+      "eval_precisions_3": 16.0769,
+      "eval_precisions_4": 10.6694,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4482,
+      "eval_rouge2": 0.2548,
+      "eval_rougeL": 0.4338,
+      "eval_rougeLsum": 0.4333,
+      "eval_runtime": 4231.6184,
+      "eval_samples_per_second": 0.521,
+      "eval_steps_per_second": 0.521,
+      "eval_sys_len": 17831,
+      "eval_totals_1": 17831,
+      "eval_totals_2": 15627,
+      "eval_totals_3": 13423,
+      "eval_totals_4": 11219,
+      "step": 654
+    },
+    {
+      "epoch": 9.99,
+      "learning_rate": 0.0001,
+      "loss": 0.8893,
+      "step": 727
+    },
+    {
+      "epoch": 9.99,
+      "eval_bleu": 19.064,
+      "eval_bp": 0.8357,
+      "eval_counts_1": 9650,
+      "eval_counts_2": 4205,
+      "eval_counts_3": 2289,
+      "eval_counts_4": 1289,
+      "eval_exact_match": 0.0463,
+      "eval_f1": 0.4472,
+      "eval_gen_len": 13.4251,
+      "eval_loss": 1.1221915483474731,
+      "eval_precisions_1": 53.5605,
+      "eval_precisions_2": 26.592,
+      "eval_precisions_3": 16.8198,
+      "eval_precisions_4": 11.3021,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4543,
+      "eval_rouge2": 0.262,
+      "eval_rougeL": 0.4396,
+      "eval_rougeLsum": 0.4394,
+      "eval_runtime": 4369.7974,
+      "eval_samples_per_second": 0.504,
+      "eval_steps_per_second": 0.504,
+      "eval_sys_len": 18017,
+      "eval_totals_1": 18017,
+      "eval_totals_2": 15813,
+      "eval_totals_3": 13609,
+      "eval_totals_4": 11405,
+      "step": 727
+    },
+    {
+      "epoch": 10.99,
+      "learning_rate": 0.0001,
+      "loss": 0.8362,
+      "step": 800
+    },
+    {
+      "epoch": 10.99,
+      "eval_bleu": 19.052,
+      "eval_bp": 0.8474,
+      "eval_counts_1": 9706,
+      "eval_counts_2": 4232,
+      "eval_counts_3": 2279,
+      "eval_counts_4": 1281,
+      "eval_exact_match": 0.0472,
+      "eval_f1": 0.4473,
+      "eval_gen_len": 13.6021,
+      "eval_loss": 1.1342219114303589,
+      "eval_precisions_1": 53.2361,
+      "eval_precisions_2": 26.4038,
+      "eval_precisions_3": 16.4858,
+      "eval_precisions_4": 11.0241,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4551,
+      "eval_rouge2": 0.2632,
+      "eval_rougeL": 0.4395,
+      "eval_rougeLsum": 0.4393,
+      "eval_runtime": 4741.4712,
+      "eval_samples_per_second": 0.465,
+      "eval_steps_per_second": 0.465,
+      "eval_sys_len": 18232,
+      "eval_totals_1": 18232,
+      "eval_totals_2": 16028,
+      "eval_totals_3": 13824,
+      "eval_totals_4": 11620,
+      "step": 800
+    },
+    {
+      "epoch": 12.0,
+      "learning_rate": 0.0001,
+      "loss": 0.7835,
+      "step": 873
+    },
+    {
+      "epoch": 12.0,
+      "eval_bleu": 19.169,
+      "eval_bp": 0.8614,
+      "eval_counts_1": 9802,
+      "eval_counts_2": 4280,
+      "eval_counts_3": 2292,
+      "eval_counts_4": 1285,
+      "eval_exact_match": 0.0472,
+      "eval_f1": 0.4497,
+      "eval_gen_len": 14.0168,
+      "eval_loss": 1.1426819562911987,
+      "eval_precisions_1": 53.0096,
+      "eval_precisions_2": 26.2786,
+      "eval_precisions_3": 16.2749,
+      "eval_precisions_4": 10.8174,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.458,
+      "eval_rouge2": 0.2634,
+      "eval_rougeL": 0.4414,
+      "eval_rougeLsum": 0.4412,
+      "eval_runtime": 2858.9204,
+      "eval_samples_per_second": 0.771,
+      "eval_steps_per_second": 0.771,
+      "eval_sys_len": 18491,
+      "eval_totals_1": 18491,
+      "eval_totals_2": 16287,
+      "eval_totals_3": 14083,
+      "eval_totals_4": 11879,
+      "step": 873
+    },
+    {
+      "epoch": 12.99,
+      "learning_rate": 0.0001,
+      "loss": 0.7441,
+      "step": 945
+    },
+    {
+      "epoch": 12.99,
+      "eval_bleu": 19.3443,
+      "eval_bp": 0.8618,
+      "eval_counts_1": 9816,
+      "eval_counts_2": 4323,
+      "eval_counts_3": 2334,
+      "eval_counts_4": 1294,
+      "eval_exact_match": 0.0463,
+      "eval_f1": 0.4493,
+      "eval_gen_len": 13.8348,
+      "eval_loss": 1.1669002771377563,
+      "eval_precisions_1": 53.0652,
+      "eval_precisions_2": 26.5312,
+      "eval_precisions_3": 16.5649,
+      "eval_precisions_4": 10.8868,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4577,
+      "eval_rouge2": 0.2659,
+      "eval_rougeL": 0.4418,
+      "eval_rougeLsum": 0.4417,
+      "eval_runtime": 2130.8,
+      "eval_samples_per_second": 1.034,
+      "eval_steps_per_second": 1.034,
+      "eval_sys_len": 18498,
+      "eval_totals_1": 18498,
+      "eval_totals_2": 16294,
+      "eval_totals_3": 14090,
+      "eval_totals_4": 11886,
+      "step": 945
+    },
+    {
+      "epoch": 13.99,
+      "learning_rate": 0.0001,
+      "loss": 0.7012,
+      "step": 1018
+    },
+    {
+      "epoch": 13.99,
+      "eval_bleu": 19.7341,
+      "eval_bp": 0.8639,
+      "eval_counts_1": 9856,
+      "eval_counts_2": 4364,
+      "eval_counts_3": 2375,
+      "eval_counts_4": 1360,
+      "eval_exact_match": 0.0476,
+      "eval_f1": 0.4514,
+      "eval_gen_len": 13.976,
+      "eval_loss": 1.1739834547042847,
+      "eval_precisions_1": 53.1693,
+      "eval_precisions_2": 26.7189,
+      "eval_precisions_3": 16.8094,
+      "eval_precisions_4": 11.4046,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4591,
+      "eval_rouge2": 0.2653,
+      "eval_rougeL": 0.443,
+      "eval_rougeLsum": 0.4428,
+      "eval_runtime": 2149.1056,
+      "eval_samples_per_second": 1.026,
+      "eval_steps_per_second": 1.026,
+      "eval_sys_len": 18537,
+      "eval_totals_1": 18537,
+      "eval_totals_2": 16333,
+      "eval_totals_3": 14129,
+      "eval_totals_4": 11925,
+      "step": 1018
+    },
+    {
+      "epoch": 14.99,
+      "learning_rate": 0.0001,
+      "loss": 0.6597,
+      "step": 1091
+    },
+    {
+      "epoch": 14.99,
+      "eval_bleu": 19.3289,
+      "eval_bp": 0.8602,
+      "eval_counts_1": 9780,
+      "eval_counts_2": 4292,
+      "eval_counts_3": 2336,
+      "eval_counts_4": 1302,
+      "eval_exact_match": 0.0485,
+      "eval_f1": 0.4492,
+      "eval_gen_len": 13.8802,
+      "eval_loss": 1.1987030506134033,
+      "eval_precisions_1": 52.9565,
+      "eval_precisions_2": 26.3896,
+      "eval_precisions_3": 16.6145,
+      "eval_precisions_4": 10.9818,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.457,
+      "eval_rouge2": 0.2633,
+      "eval_rougeL": 0.4418,
+      "eval_rougeLsum": 0.4416,
+      "eval_runtime": 2149.2833,
+      "eval_samples_per_second": 1.025,
+      "eval_steps_per_second": 1.025,
+      "eval_sys_len": 18468,
+      "eval_totals_1": 18468,
+      "eval_totals_2": 16264,
+      "eval_totals_3": 14060,
+      "eval_totals_4": 11856,
+      "step": 1091
+    },
+    {
+      "epoch": 16.0,
+      "learning_rate": 0.0001,
+      "loss": 0.6236,
+      "step": 1164
+    },
+    {
+      "epoch": 16.0,
+      "eval_bleu": 19.8055,
+      "eval_bp": 0.8734,
+      "eval_counts_1": 9931,
+      "eval_counts_2": 4388,
+      "eval_counts_3": 2390,
+      "eval_counts_4": 1359,
+      "eval_exact_match": 0.0495,
+      "eval_f1": 0.4538,
+      "eval_gen_len": 14.044,
+      "eval_loss": 1.2135030031204224,
+      "eval_precisions_1": 53.0587,
+      "eval_precisions_2": 26.573,
+      "eval_precisions_3": 16.7028,
+      "eval_precisions_4": 11.2268,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4618,
+      "eval_rouge2": 0.2682,
+      "eval_rougeL": 0.4452,
+      "eval_rougeLsum": 0.445,
+      "eval_runtime": 2168.1341,
+      "eval_samples_per_second": 1.017,
+      "eval_steps_per_second": 1.017,
+      "eval_sys_len": 18717,
+      "eval_totals_1": 18717,
+      "eval_totals_2": 16513,
+      "eval_totals_3": 14309,
+      "eval_totals_4": 12105,
+      "step": 1164
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 0.0001,
+      "loss": 0.5933,
+      "step": 1237
+    },
+    {
+      "epoch": 17.0,
+      "eval_bleu": 19.5893,
+      "eval_bp": 0.8654,
+      "eval_counts_1": 9806,
+      "eval_counts_2": 4316,
+      "eval_counts_3": 2366,
+      "eval_counts_4": 1348,
+      "eval_exact_match": 0.049,
+      "eval_f1": 0.4485,
+      "eval_gen_len": 14.0622,
+      "eval_loss": 1.2305341958999634,
+      "eval_precisions_1": 52.817,
+      "eval_precisions_2": 26.3782,
+      "eval_precisions_3": 16.7114,
+      "eval_precisions_4": 11.2766,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4571,
+      "eval_rouge2": 0.2628,
+      "eval_rougeL": 0.4407,
+      "eval_rougeLsum": 0.4409,
+      "eval_runtime": 2171.7325,
+      "eval_samples_per_second": 1.015,
+      "eval_steps_per_second": 1.015,
+      "eval_sys_len": 18566,
+      "eval_totals_1": 18566,
+      "eval_totals_2": 16362,
+      "eval_totals_3": 14158,
+      "eval_totals_4": 11954,
+      "step": 1237
+    },
+    {
+      "epoch": 17.99,
+      "learning_rate": 0.0001,
+      "loss": 0.5622,
+      "step": 1309
+    },
+    {
+      "epoch": 17.99,
+      "eval_bleu": 19.4914,
+      "eval_bp": 0.865,
+      "eval_counts_1": 9787,
+      "eval_counts_2": 4306,
+      "eval_counts_3": 2346,
+      "eval_counts_4": 1338,
+      "eval_exact_match": 0.0476,
+      "eval_f1": 0.447,
+      "eval_gen_len": 13.7763,
+      "eval_loss": 1.2796473503112793,
+      "eval_precisions_1": 52.7345,
+      "eval_precisions_2": 26.3283,
+      "eval_precisions_3": 16.5783,
+      "eval_precisions_4": 11.1995,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4549,
+      "eval_rouge2": 0.2609,
+      "eval_rougeL": 0.4383,
+      "eval_rougeLsum": 0.4382,
+      "eval_runtime": 2158.5699,
+      "eval_samples_per_second": 1.021,
+      "eval_steps_per_second": 1.021,
+      "eval_sys_len": 18559,
+      "eval_totals_1": 18559,
+      "eval_totals_2": 16355,
+      "eval_totals_3": 14151,
+      "eval_totals_4": 11947,
+      "step": 1309
+    },
+    {
+      "epoch": 18.99,
+      "learning_rate": 0.0001,
+      "loss": 0.5275,
+      "step": 1382
+    },
+    {
+      "epoch": 18.99,
+      "eval_bleu": 19.6947,
+      "eval_bp": 0.8857,
+      "eval_counts_1": 9918,
+      "eval_counts_2": 4363,
+      "eval_counts_3": 2374,
+      "eval_counts_4": 1355,
+      "eval_exact_match": 0.0508,
+      "eval_f1": 0.4499,
+      "eval_gen_len": 14.1647,
+      "eval_loss": 1.2833356857299805,
+      "eval_precisions_1": 52.3377,
+      "eval_precisions_2": 26.054,
+      "eval_precisions_3": 16.3251,
+      "eval_precisions_4": 10.9823,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4573,
+      "eval_rouge2": 0.2624,
+      "eval_rougeL": 0.441,
+      "eval_rougeLsum": 0.4408,
+      "eval_runtime": 2190.1704,
+      "eval_samples_per_second": 1.006,
+      "eval_steps_per_second": 1.006,
+      "eval_sys_len": 18950,
+      "eval_totals_1": 18950,
+      "eval_totals_2": 16746,
+      "eval_totals_3": 14542,
+      "eval_totals_4": 12338,
+      "step": 1382
+    },
+    {
+      "epoch": 19.79,
+      "learning_rate": 0.0001,
+      "loss": 0.4986,
+      "step": 1440
+    },
+    {
+      "epoch": 19.79,
+      "eval_bleu": 19.4544,
+      "eval_bp": 0.8847,
+      "eval_counts_1": 9879,
+      "eval_counts_2": 4315,
+      "eval_counts_3": 2347,
+      "eval_counts_4": 1324,
+      "eval_exact_match": 0.0495,
+      "eval_f1": 0.4478,
+      "eval_gen_len": 14.2827,
+      "eval_loss": 1.3059108257293701,
+      "eval_precisions_1": 52.1842,
+      "eval_precisions_2": 25.7966,
+      "eval_precisions_3": 16.1606,
+      "eval_precisions_4": 10.7476,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4564,
+      "eval_rouge2": 0.2622,
+      "eval_rougeL": 0.4407,
+      "eval_rougeLsum": 0.4403,
+      "eval_runtime": 3646.8693,
+      "eval_samples_per_second": 0.604,
+      "eval_steps_per_second": 0.604,
+      "eval_sys_len": 18931,
+      "eval_totals_1": 18931,
+      "eval_totals_2": 16727,
+      "eval_totals_3": 14523,
+      "eval_totals_4": 12319,
+      "step": 1440
+    },
+    {
+      "epoch": 19.79,
+      "step": 1440,
+      "total_flos": 1.102412878184448e+18,
+      "train_loss": 1.0667428798145717,
+      "train_runtime": 140813.6912,
+      "train_samples_per_second": 1.323,
+      "train_steps_per_second": 0.01
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 1440,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "total_flos": 1.102412878184448e+18,
+  "trial_name": null,
+  "trial_params": null
+}