Training in progress, epoch 0

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,64 @@
+---
+base_model: google/gemma-2-9b-it
+datasets:
+- generator
+library_name: peft
+license: gemma
+tags:
+- trl
+- sft
+- generated_from_trainer
+model-index:
+- name: gemma-2-9b-it-lora-magicoder
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-2-9b-it-lora-magicoder
+
+This model is a fine-tuned version of [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) on the alvarobartt/Magicoder-OAI dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.12.0
+- Transformers 4.44.2
+- Pytorch 2.3.0+cu121
+- Datasets 3.0.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-2-2b-it",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.01,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97d5e52b5668436977b4675922d9a5540ecdfc875c8488911bbe650cec148de2
+size 41582088

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.997420464316423,
+    "total_flos": 1.7624457120055296e+16,
+    "train_loss": 0.4506890232783245,
+    "train_runtime": 9589.9992,
+    "train_samples": 67677,
+    "train_samples_per_second": 5.818,
+    "train_steps_per_second": 0.182
+}

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "google/gemma-2-9b-it",
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 256000
+}

special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f289bc05132635a8bc7aca7aa21255efd5e18f3710f43e3cdb96bcd41be4922
+size 17525357

tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "model_max_length": 2048,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.997420464316423,
+    "total_flos": 1.7624457120055296e+16,
+    "train_loss": 0.4506890232783245,
+    "train_runtime": 9589.9992,
+    "train_samples": 67677,
+    "train_samples_per_second": 5.818,
+    "train_steps_per_second": 0.182
+}

trainer_state.json

@@ -0,0 +1,1260 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.997420464316423,
+  "eval_steps": 500,
+  "global_step": 1743,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017196904557179708,
+      "grad_norm": 1.236184773002013,
+      "learning_rate": 1.1428571428571429e-05,
+      "loss": 1.185,
+      "step": 10
+    },
+    {
+      "epoch": 0.034393809114359415,
+      "grad_norm": 0.5456076333168367,
+      "learning_rate": 2.2857142857142858e-05,
+      "loss": 1.0131,
+      "step": 20
+    },
+    {
+      "epoch": 0.051590713671539126,
+      "grad_norm": 0.35549053304697176,
+      "learning_rate": 3.428571428571429e-05,
+      "loss": 0.8296,
+      "step": 30
+    },
+    {
+      "epoch": 0.06878761822871883,
+      "grad_norm": 0.254673976618386,
+      "learning_rate": 4.5714285714285716e-05,
+      "loss": 0.7454,
+      "step": 40
+    },
+    {
+      "epoch": 0.08598452278589853,
+      "grad_norm": 0.23070603652224944,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 0.6662,
+      "step": 50
+    },
+    {
+      "epoch": 0.10318142734307825,
+      "grad_norm": 0.17478929504818824,
+      "learning_rate": 6.857142857142858e-05,
+      "loss": 0.596,
+      "step": 60
+    },
+    {
+      "epoch": 0.12037833190025796,
+      "grad_norm": 0.1561819886139925,
+      "learning_rate": 8e-05,
+      "loss": 0.578,
+      "step": 70
+    },
+    {
+      "epoch": 0.13757523645743766,
+      "grad_norm": 0.20767032014128362,
+      "learning_rate": 9.142857142857143e-05,
+      "loss": 0.56,
+      "step": 80
+    },
+    {
+      "epoch": 0.15477214101461736,
+      "grad_norm": 0.1783753453274605,
+      "learning_rate": 0.00010285714285714286,
+      "loss": 0.5486,
+      "step": 90
+    },
+    {
+      "epoch": 0.17196904557179707,
+      "grad_norm": 0.21780447055549776,
+      "learning_rate": 0.00011428571428571428,
+      "loss": 0.5442,
+      "step": 100
+    },
+    {
+      "epoch": 0.18916595012897677,
+      "grad_norm": 0.17636411695201717,
+      "learning_rate": 0.00012571428571428572,
+      "loss": 0.5336,
+      "step": 110
+    },
+    {
+      "epoch": 0.2063628546861565,
+      "grad_norm": 0.17514613701190404,
+      "learning_rate": 0.00013714285714285716,
+      "loss": 0.5267,
+      "step": 120
+    },
+    {
+      "epoch": 0.2235597592433362,
+      "grad_norm": 0.19025963821244604,
+      "learning_rate": 0.00014857142857142857,
+      "loss": 0.5274,
+      "step": 130
+    },
+    {
+      "epoch": 0.2407566638005159,
+      "grad_norm": 0.18019488365092756,
+      "learning_rate": 0.00016,
+      "loss": 0.5166,
+      "step": 140
+    },
+    {
+      "epoch": 0.2579535683576956,
+      "grad_norm": 0.17880762343061918,
+      "learning_rate": 0.00017142857142857143,
+      "loss": 0.5203,
+      "step": 150
+    },
+    {
+      "epoch": 0.2751504729148753,
+      "grad_norm": 0.18388862726424096,
+      "learning_rate": 0.00018285714285714286,
+      "loss": 0.5224,
+      "step": 160
+    },
+    {
+      "epoch": 0.292347377472055,
+      "grad_norm": 0.16430698379785788,
+      "learning_rate": 0.0001942857142857143,
+      "loss": 0.5188,
+      "step": 170
+    },
+    {
+      "epoch": 0.30954428202923473,
+      "grad_norm": 0.15725107599349494,
+      "learning_rate": 0.00019999498219234568,
+      "loss": 0.516,
+      "step": 180
+    },
+    {
+      "epoch": 0.32674118658641443,
+      "grad_norm": 0.15173489549988936,
+      "learning_rate": 0.0001999548427524678,
+      "loss": 0.5058,
+      "step": 190
+    },
+    {
+      "epoch": 0.34393809114359414,
+      "grad_norm": 0.16657669365074929,
+      "learning_rate": 0.0001998745799852668,
+      "loss": 0.5072,
+      "step": 200
+    },
+    {
+      "epoch": 0.36113499570077384,
+      "grad_norm": 0.14245522126193838,
+      "learning_rate": 0.00019975422610938462,
+      "loss": 0.5092,
+      "step": 210
+    },
+    {
+      "epoch": 0.37833190025795355,
+      "grad_norm": 0.14204763106782423,
+      "learning_rate": 0.00019959382943661704,
+      "loss": 0.5124,
+      "step": 220
+    },
+    {
+      "epoch": 0.39552880481513325,
+      "grad_norm": 0.14674244927319882,
+      "learning_rate": 0.00019939345435252088,
+      "loss": 0.515,
+      "step": 230
+    },
+    {
+      "epoch": 0.412725709372313,
+      "grad_norm": 0.13843089052767493,
+      "learning_rate": 0.00019915318129056853,
+      "loss": 0.4994,
+      "step": 240
+    },
+    {
+      "epoch": 0.4299226139294927,
+      "grad_norm": 0.146900895726969,
+      "learning_rate": 0.00019887310669986085,
+      "loss": 0.5052,
+      "step": 250
+    },
+    {
+      "epoch": 0.4471195184866724,
+      "grad_norm": 0.14008859218618483,
+      "learning_rate": 0.00019855334300641114,
+      "loss": 0.5041,
+      "step": 260
+    },
+    {
+      "epoch": 0.4643164230438521,
+      "grad_norm": 0.18809570469767778,
+      "learning_rate": 0.0001981940185680156,
+      "loss": 0.4978,
+      "step": 270
+    },
+    {
+      "epoch": 0.4815133276010318,
+      "grad_norm": 0.14104099043343712,
+      "learning_rate": 0.00019779527762272877,
+      "loss": 0.5001,
+      "step": 280
+    },
+    {
+      "epoch": 0.49871023215821153,
+      "grad_norm": 0.13804026353342516,
+      "learning_rate": 0.0001973572802309642,
+      "loss": 0.4885,
+      "step": 290
+    },
+    {
+      "epoch": 0.5159071367153912,
+      "grad_norm": 0.14300868500069924,
+      "learning_rate": 0.00019688020221124376,
+      "loss": 0.495,
+      "step": 300
+    },
+    {
+      "epoch": 0.5331040412725709,
+      "grad_norm": 0.14885187886758067,
+      "learning_rate": 0.00019636423506962181,
+      "loss": 0.4963,
+      "step": 310
+    },
+    {
+      "epoch": 0.5503009458297506,
+      "grad_norm": 0.13442343336925347,
+      "learning_rate": 0.00019580958592281167,
+      "loss": 0.4924,
+      "step": 320
+    },
+    {
+      "epoch": 0.5674978503869303,
+      "grad_norm": 0.13673211813600614,
+      "learning_rate": 0.00019521647741504604,
+      "loss": 0.5004,
+      "step": 330
+    },
+    {
+      "epoch": 0.58469475494411,
+      "grad_norm": 0.13956825916960136,
+      "learning_rate": 0.00019458514762870426,
+      "loss": 0.485,
+      "step": 340
+    },
+    {
+      "epoch": 0.6018916595012898,
+      "grad_norm": 0.14073396199862337,
+      "learning_rate": 0.0001939158499887428,
+      "loss": 0.4933,
+      "step": 350
+    },
+    {
+      "epoch": 0.6190885640584695,
+      "grad_norm": 0.13603301551856595,
+      "learning_rate": 0.00019320885316096654,
+      "loss": 0.4938,
+      "step": 360
+    },
+    {
+      "epoch": 0.6362854686156492,
+      "grad_norm": 0.1420552122294903,
+      "learning_rate": 0.00019246444094418255,
+      "loss": 0.4786,
+      "step": 370
+    },
+    {
+      "epoch": 0.6534823731728289,
+      "grad_norm": 0.1414519600117056,
+      "learning_rate": 0.00019168291215627926,
+      "loss": 0.4926,
+      "step": 380
+    },
+    {
+      "epoch": 0.6706792777300086,
+      "grad_norm": 0.13984821923297133,
+      "learning_rate": 0.00019086458051427622,
+      "loss": 0.4911,
+      "step": 390
+    },
+    {
+      "epoch": 0.6878761822871883,
+      "grad_norm": 0.13752756195301288,
+      "learning_rate": 0.00019000977450839393,
+      "loss": 0.4827,
+      "step": 400
+    },
+    {
+      "epoch": 0.705073086844368,
+      "grad_norm": 0.13335244333926366,
+      "learning_rate": 0.00018911883727019285,
+      "loss": 0.4898,
+      "step": 410
+    },
+    {
+      "epoch": 0.7222699914015477,
+      "grad_norm": 0.13230952803690713,
+      "learning_rate": 0.0001881921264348355,
+      "loss": 0.4854,
+      "step": 420
+    },
+    {
+      "epoch": 0.7394668959587274,
+      "grad_norm": 0.13478694398925248,
+      "learning_rate": 0.00018723001399752653,
+      "loss": 0.4897,
+      "step": 430
+    },
+    {
+      "epoch": 0.7566638005159071,
+      "grad_norm": 0.1442034436213339,
+      "learning_rate": 0.0001862328861641883,
+      "loss": 0.4822,
+      "step": 440
+    },
+    {
+      "epoch": 0.7738607050730868,
+      "grad_norm": 0.13449084966209246,
+      "learning_rate": 0.00018520114319643235,
+      "loss": 0.4886,
+      "step": 450
+    },
+    {
+      "epoch": 0.7910576096302665,
+      "grad_norm": 0.1312938599246684,
+      "learning_rate": 0.0001841351992508885,
+      "loss": 0.4786,
+      "step": 460
+    },
+    {
+      "epoch": 0.8082545141874462,
+      "grad_norm": 0.12664552711595364,
+      "learning_rate": 0.0001830354822129564,
+      "loss": 0.4882,
+      "step": 470
+    },
+    {
+      "epoch": 0.825451418744626,
+      "grad_norm": 0.134819817437524,
+      "learning_rate": 0.00018190243352504597,
+      "loss": 0.48,
+      "step": 480
+    },
+    {
+      "epoch": 0.8426483233018057,
+      "grad_norm": 0.14466404063388616,
+      "learning_rate": 0.00018073650800937624,
+      "loss": 0.4809,
+      "step": 490
+    },
+    {
+      "epoch": 0.8598452278589854,
+      "grad_norm": 0.13311350418133092,
+      "learning_rate": 0.00017953817368540292,
+      "loss": 0.4834,
+      "step": 500
+    },
+    {
+      "epoch": 0.8770421324161651,
+      "grad_norm": 0.13856027344204028,
+      "learning_rate": 0.0001783079115819486,
+      "loss": 0.4913,
+      "step": 510
+    },
+    {
+      "epoch": 0.8942390369733448,
+      "grad_norm": 0.12870004634007948,
+      "learning_rate": 0.00017704621554411084,
+      "loss": 0.4827,
+      "step": 520
+    },
+    {
+      "epoch": 0.9114359415305245,
+      "grad_norm": 0.12894303718503525,
+      "learning_rate": 0.0001757535920350255,
+      "loss": 0.4768,
+      "step": 530
+    },
+    {
+      "epoch": 0.9286328460877042,
+      "grad_norm": 0.13086927730335407,
+      "learning_rate": 0.0001744305599325652,
+      "loss": 0.4857,
+      "step": 540
+    },
+    {
+      "epoch": 0.945829750644884,
+      "grad_norm": 0.12929634161460313,
+      "learning_rate": 0.00017307765032105406,
+      "loss": 0.4883,
+      "step": 550
+    },
+    {
+      "epoch": 0.9630266552020637,
+      "grad_norm": 0.13721669025539573,
+      "learning_rate": 0.00017169540627808274,
+      "loss": 0.4875,
+      "step": 560
+    },
+    {
+      "epoch": 0.9802235597592434,
+      "grad_norm": 0.1285154415557853,
+      "learning_rate": 0.00017028438265650933,
+      "loss": 0.4821,
+      "step": 570
+    },
+    {
+      "epoch": 0.9974204643164231,
+      "grad_norm": 0.13060285124046392,
+      "learning_rate": 0.0001688451458617332,
+      "loss": 0.4796,
+      "step": 580
+    },
+    {
+      "epoch": 1.0146173688736027,
+      "grad_norm": 0.136066972815243,
+      "learning_rate": 0.00016737827362433164,
+      "loss": 0.4527,
+      "step": 590
+    },
+    {
+      "epoch": 1.0318142734307825,
+      "grad_norm": 0.1366031218446295,
+      "learning_rate": 0.0001658843547681506,
+      "loss": 0.4484,
+      "step": 600
+    },
+    {
+      "epoch": 1.049011177987962,
+      "grad_norm": 0.1378157386033563,
+      "learning_rate": 0.000164363988973942,
+      "loss": 0.4465,
+      "step": 610
+    },
+    {
+      "epoch": 1.0662080825451419,
+      "grad_norm": 0.1435554531245416,
+      "learning_rate": 0.00016281778653864316,
+      "loss": 0.4415,
+      "step": 620
+    },
+    {
+      "epoch": 1.0834049871023215,
+      "grad_norm": 0.14661925606567783,
+      "learning_rate": 0.00016124636813039502,
+      "loss": 0.441,
+      "step": 630
+    },
+    {
+      "epoch": 1.1006018916595013,
+      "grad_norm": 0.13987832152903712,
+      "learning_rate": 0.0001596503645393966,
+      "loss": 0.4359,
+      "step": 640
+    },
+    {
+      "epoch": 1.117798796216681,
+      "grad_norm": 0.14735126394013195,
+      "learning_rate": 0.0001580304164246968,
+      "loss": 0.4473,
+      "step": 650
+    },
+    {
+      "epoch": 1.1349957007738607,
+      "grad_norm": 0.1429953902523938,
+      "learning_rate": 0.0001563871740570245,
+      "loss": 0.4451,
+      "step": 660
+    },
+    {
+      "epoch": 1.1521926053310405,
+      "grad_norm": 0.1459735378688384,
+      "learning_rate": 0.00015472129705776047,
+      "loss": 0.4422,
+      "step": 670
+    },
+    {
+      "epoch": 1.16938950988822,
+      "grad_norm": 0.14561658254102702,
+      "learning_rate": 0.00015303345413415564,
+      "loss": 0.4409,
+      "step": 680
+    },
+    {
+      "epoch": 1.1865864144454,
+      "grad_norm": 0.14737248104484174,
+      "learning_rate": 0.00015132432281090256,
+      "loss": 0.4419,
+      "step": 690
+    },
+    {
+      "epoch": 1.2037833190025795,
+      "grad_norm": 0.15311335569967213,
+      "learning_rate": 0.0001495945891581668,
+      "loss": 0.4419,
+      "step": 700
+    },
+    {
+      "epoch": 1.2209802235597593,
+      "grad_norm": 0.14524361407042563,
+      "learning_rate": 0.00014784494751618853,
+      "loss": 0.4411,
+      "step": 710
+    },
+    {
+      "epoch": 1.238177128116939,
+      "grad_norm": 0.14910830182050996,
+      "learning_rate": 0.0001460761002165645,
+      "loss": 0.4471,
+      "step": 720
+    },
+    {
+      "epoch": 1.2553740326741187,
+      "grad_norm": 0.15226336213789984,
+      "learning_rate": 0.00014428875730032145,
+      "loss": 0.4423,
+      "step": 730
+    },
+    {
+      "epoch": 1.2725709372312983,
+      "grad_norm": 0.15781539164450759,
+      "learning_rate": 0.00014248363623289574,
+      "loss": 0.4359,
+      "step": 740
+    },
+    {
+      "epoch": 1.2897678417884781,
+      "grad_norm": 0.14138385096885275,
+      "learning_rate": 0.00014066146161613208,
+      "loss": 0.4414,
+      "step": 750
+    },
+    {
+      "epoch": 1.3069647463456577,
+      "grad_norm": 0.14745231010770607,
+      "learning_rate": 0.00013882296489741783,
+      "loss": 0.4457,
+      "step": 760
+    },
+    {
+      "epoch": 1.3241616509028376,
+      "grad_norm": 0.155323506204182,
+      "learning_rate": 0.00013696888407606952,
+      "loss": 0.4468,
+      "step": 770
+    },
+    {
+      "epoch": 1.3413585554600171,
+      "grad_norm": 0.14906188336013929,
+      "learning_rate": 0.0001350999634070889,
+      "loss": 0.4561,
+      "step": 780
+    },
+    {
+      "epoch": 1.358555460017197,
+      "grad_norm": 0.1509482546916061,
+      "learning_rate": 0.0001332169531024085,
+      "loss": 0.4488,
+      "step": 790
+    },
+    {
+      "epoch": 1.3757523645743766,
+      "grad_norm": 0.1577110542147225,
+      "learning_rate": 0.00013132060902974554,
+      "loss": 0.4481,
+      "step": 800
+    },
+    {
+      "epoch": 1.3929492691315564,
+      "grad_norm": 0.1519302579308639,
+      "learning_rate": 0.00012941169240918534,
+      "loss": 0.4434,
+      "step": 810
+    },
+    {
+      "epoch": 1.410146173688736,
+      "grad_norm": 0.16574786040919845,
+      "learning_rate": 0.00012749096950761702,
+      "loss": 0.4422,
+      "step": 820
+    },
+    {
+      "epoch": 1.4273430782459158,
+      "grad_norm": 0.14939058136631353,
+      "learning_rate": 0.00012555921133114247,
+      "loss": 0.4439,
+      "step": 830
+    },
+    {
+      "epoch": 1.4445399828030954,
+      "grad_norm": 0.14433259289852834,
+      "learning_rate": 0.00012361719331558345,
+      "loss": 0.4412,
+      "step": 840
+    },
+    {
+      "epoch": 1.4617368873602752,
+      "grad_norm": 0.14779220063991433,
+      "learning_rate": 0.00012166569501521017,
+      "loss": 0.444,
+      "step": 850
+    },
+    {
+      "epoch": 1.4789337919174548,
+      "grad_norm": 0.15109790098843823,
+      "learning_rate": 0.00011970549978981715,
+      "loss": 0.4489,
+      "step": 860
+    },
+    {
+      "epoch": 1.4961306964746346,
+      "grad_norm": 0.14824981470720835,
+      "learning_rate": 0.00011773739449027108,
+      "loss": 0.4377,
+      "step": 870
+    },
+    {
+      "epoch": 1.5133276010318144,
+      "grad_norm": 0.14490199589122096,
+      "learning_rate": 0.00011576216914265734,
+      "loss": 0.4454,
+      "step": 880
+    },
+    {
+      "epoch": 1.530524505588994,
+      "grad_norm": 0.14668336107167357,
+      "learning_rate": 0.00011378061663115222,
+      "loss": 0.4374,
+      "step": 890
+    },
+    {
+      "epoch": 1.5477214101461736,
+      "grad_norm": 0.15306161877683747,
+      "learning_rate": 0.00011179353237974756,
+      "loss": 0.441,
+      "step": 900
+    },
+    {
+      "epoch": 1.5649183147033534,
+      "grad_norm": 0.14629253984953647,
+      "learning_rate": 0.0001098017140329561,
+      "loss": 0.4402,
+      "step": 910
+    },
+    {
+      "epoch": 1.5821152192605332,
+      "grad_norm": 0.15181211231388803,
+      "learning_rate": 0.00010780596113562514,
+      "loss": 0.4441,
+      "step": 920
+    },
+    {
+      "epoch": 1.5993121238177128,
+      "grad_norm": 0.14895949287614227,
+      "learning_rate": 0.00010580707481198796,
+      "loss": 0.4415,
+      "step": 930
+    },
+    {
+      "epoch": 1.6165090283748924,
+      "grad_norm": 0.14919588155250404,
+      "learning_rate": 0.00010380585744408065,
+      "loss": 0.4428,
+      "step": 940
+    },
+    {
+      "epoch": 1.6337059329320722,
+      "grad_norm": 0.15195396903926595,
+      "learning_rate": 0.00010180311234965433,
+      "loss": 0.4338,
+      "step": 950
+    },
+    {
+      "epoch": 1.650902837489252,
+      "grad_norm": 0.15192117531451788,
+      "learning_rate": 9.979964345971188e-05,
+      "loss": 0.445,
+      "step": 960
+    },
+    {
+      "epoch": 1.6680997420464316,
+      "grad_norm": 0.14674515514268918,
+      "learning_rate": 9.779625499579805e-05,
+      "loss": 0.4416,
+      "step": 970
+    },
+    {
+      "epoch": 1.6852966466036112,
+      "grad_norm": 0.14766438549971136,
+      "learning_rate": 9.579375114717351e-05,
+      "loss": 0.434,
+      "step": 980
+    },
+    {
+      "epoch": 1.702493551160791,
+      "grad_norm": 0.14681911182175214,
+      "learning_rate": 9.379293574800154e-05,
+      "loss": 0.4363,
+      "step": 990
+    },
+    {
+      "epoch": 1.7196904557179709,
+      "grad_norm": 0.1464492709373849,
+      "learning_rate": 9.179461195467714e-05,
+      "loss": 0.4353,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7368873602751504,
+      "grad_norm": 0.15011221390319449,
+      "learning_rate": 8.979958192342862e-05,
+      "loss": 0.4445,
+      "step": 1010
+    },
+    {
+      "epoch": 1.75408426483233,
+      "grad_norm": 0.152429302734013,
+      "learning_rate": 8.780864648832022e-05,
+      "loss": 0.4404,
+      "step": 1020
+    },
+    {
+      "epoch": 1.7712811693895099,
+      "grad_norm": 0.14681226342261067,
+      "learning_rate": 8.58226048397857e-05,
+      "loss": 0.4435,
+      "step": 1030
+    },
+    {
+      "epoch": 1.7884780739466897,
+      "grad_norm": 0.15354804740449426,
+      "learning_rate": 8.384225420382185e-05,
+      "loss": 0.4377,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8056749785038693,
+      "grad_norm": 0.15108728472484245,
+      "learning_rate": 8.186838952197018e-05,
+      "loss": 0.4344,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8228718830610489,
+      "grad_norm": 0.14903695640608858,
+      "learning_rate": 7.990180313221596e-05,
+      "loss": 0.4381,
+      "step": 1060
+    },
+    {
+      "epoch": 1.8400687876182287,
+      "grad_norm": 0.14676064171835554,
+      "learning_rate": 7.794328445093208e-05,
+      "loss": 0.4379,
+      "step": 1070
+    },
+    {
+      "epoch": 1.8572656921754085,
+      "grad_norm": 0.15258822928003124,
+      "learning_rate": 7.599361965599606e-05,
+      "loss": 0.4426,
+      "step": 1080
+    },
+    {
+      "epoch": 1.874462596732588,
+      "grad_norm": 0.26094072520435535,
+      "learning_rate": 7.405359137120662e-05,
+      "loss": 0.4368,
+      "step": 1090
+    },
+    {
+      "epoch": 1.8916595012897677,
+      "grad_norm": 0.15223526485978567,
+      "learning_rate": 7.212397835212722e-05,
+      "loss": 0.4426,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9088564058469477,
+      "grad_norm": 0.14158786202820334,
+      "learning_rate": 7.02055551734822e-05,
+      "loss": 0.4304,
+      "step": 1110
+    },
+    {
+      "epoch": 1.9260533104041273,
+      "grad_norm": 0.15562496727433017,
+      "learning_rate": 6.829909191823121e-05,
+      "loss": 0.4426,
+      "step": 1120
+    },
+    {
+      "epoch": 1.943250214961307,
+      "grad_norm": 0.14930981803960258,
+      "learning_rate": 6.640535386844679e-05,
+      "loss": 0.4418,
+      "step": 1130
+    },
+    {
+      "epoch": 1.9604471195184867,
+      "grad_norm": 0.1552156563224316,
+      "learning_rate": 6.452510119811895e-05,
+      "loss": 0.4367,
+      "step": 1140
+    },
+    {
+      "epoch": 1.9776440240756665,
+      "grad_norm": 0.19090518551388455,
+      "learning_rate": 6.26590886680103e-05,
+      "loss": 0.4303,
+      "step": 1150
+    },
+    {
+      "epoch": 1.9948409286328461,
+      "grad_norm": 0.1496699651502689,
+      "learning_rate": 6.0808065322683993e-05,
+      "loss": 0.4307,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0120378331900257,
+      "grad_norm": 0.16615207318616432,
+      "learning_rate": 5.897277418982672e-05,
+      "loss": 0.3983,
+      "step": 1170
+    },
+    {
+      "epoch": 2.0292347377472053,
+      "grad_norm": 0.1652827612761684,
+      "learning_rate": 5.715395198198603e-05,
+      "loss": 0.3812,
+      "step": 1180
+    },
+    {
+      "epoch": 2.0464316423043853,
+      "grad_norm": 0.16911724923353538,
+      "learning_rate": 5.5352328800843724e-05,
+      "loss": 0.3766,
+      "step": 1190
+    },
+    {
+      "epoch": 2.063628546861565,
+      "grad_norm": 0.16951539871441879,
+      "learning_rate": 5.356862784414199e-05,
+      "loss": 0.3789,
+      "step": 1200
+    },
+    {
+      "epoch": 2.0808254514187445,
+      "grad_norm": 0.17010125437035936,
+      "learning_rate": 5.1803565115381694e-05,
+      "loss": 0.3783,
+      "step": 1210
+    },
+    {
+      "epoch": 2.098022355975924,
+      "grad_norm": 0.16860757374113347,
+      "learning_rate": 5.0057849136407874e-05,
+      "loss": 0.3815,
+      "step": 1220
+    },
+    {
+      "epoch": 2.115219260533104,
+      "grad_norm": 0.17279827710775011,
+      "learning_rate": 4.833218066299896e-05,
+      "loss": 0.3775,
+      "step": 1230
+    },
+    {
+      "epoch": 2.1324161650902838,
+      "grad_norm": 0.16772946452531912,
+      "learning_rate": 4.6627252403573085e-05,
+      "loss": 0.3719,
+      "step": 1240
+    },
+    {
+      "epoch": 2.1496130696474633,
+      "grad_norm": 0.17982428185526986,
+      "learning_rate": 4.4943748741124934e-05,
+      "loss": 0.3779,
+      "step": 1250
+    },
+    {
+      "epoch": 2.166809974204643,
+      "grad_norm": 0.18313593415470272,
+      "learning_rate": 4.328234545850442e-05,
+      "loss": 0.3836,
+      "step": 1260
+    },
+    {
+      "epoch": 2.184006878761823,
+      "grad_norm": 0.1725383341952266,
+      "learning_rate": 4.1643709467147615e-05,
+      "loss": 0.369,
+      "step": 1270
+    },
+    {
+      "epoch": 2.2012037833190026,
+      "grad_norm": 0.1971081873293993,
+      "learning_rate": 4.002849853936891e-05,
+      "loss": 0.3728,
+      "step": 1280
+    },
+    {
+      "epoch": 2.218400687876182,
+      "grad_norm": 0.17991148711708474,
+      "learning_rate": 3.843736104432137e-05,
+      "loss": 0.3804,
+      "step": 1290
+    },
+    {
+      "epoch": 2.235597592433362,
+      "grad_norm": 0.17934271418386846,
+      "learning_rate": 3.687093568773229e-05,
+      "loss": 0.3844,
+      "step": 1300
+    },
+    {
+      "epoch": 2.252794496990542,
+      "grad_norm": 0.17862554374418718,
+      "learning_rate": 3.532985125551715e-05,
+      "loss": 0.385,
+      "step": 1310
+    },
+    {
+      "epoch": 2.2699914015477214,
+      "grad_norm": 0.17397364598815349,
+      "learning_rate": 3.381472636137591e-05,
+      "loss": 0.3755,
+      "step": 1320
+    },
+    {
+      "epoch": 2.287188306104901,
+      "grad_norm": 0.18500269890886714,
+      "learning_rate": 3.2326169198472556e-05,
+      "loss": 0.384,
+      "step": 1330
+    },
+    {
+      "epoch": 2.304385210662081,
+      "grad_norm": 0.18186600816942164,
+      "learning_rate": 3.0864777295297376e-05,
+      "loss": 0.3726,
+      "step": 1340
+    },
+    {
+      "epoch": 2.3215821152192606,
+      "grad_norm": 0.17773737221105249,
+      "learning_rate": 2.9431137275810317e-05,
+      "loss": 0.3704,
+      "step": 1350
+    },
+    {
+      "epoch": 2.33877901977644,
+      "grad_norm": 0.18256211613697612,
+      "learning_rate": 2.8025824623961773e-05,
+      "loss": 0.3769,
+      "step": 1360
+    },
+    {
+      "epoch": 2.35597592433362,
+      "grad_norm": 0.18129287061779392,
+      "learning_rate": 2.664940345268483e-05,
+      "loss": 0.3766,
+      "step": 1370
+    },
+    {
+      "epoch": 2.3731728288908,
+      "grad_norm": 0.1785686917578723,
+      "learning_rate": 2.5302426277452172e-05,
+      "loss": 0.3701,
+      "step": 1380
+    },
+    {
+      "epoch": 2.3903697334479794,
+      "grad_norm": 0.19834019533954172,
+      "learning_rate": 2.398543379448832e-05,
+      "loss": 0.3768,
+      "step": 1390
+    },
+    {
+      "epoch": 2.407566638005159,
+      "grad_norm": 0.18177776105752433,
+      "learning_rate": 2.26989546637263e-05,
+      "loss": 0.3755,
+      "step": 1400
+    },
+    {
+      "epoch": 2.4247635425623386,
+      "grad_norm": 0.18362730161587285,
+      "learning_rate": 2.144350529659589e-05,
+      "loss": 0.368,
+      "step": 1410
+    },
+    {
+      "epoch": 2.4419604471195187,
+      "grad_norm": 0.18330397808911847,
+      "learning_rate": 2.021958964872851e-05,
+      "loss": 0.3794,
+      "step": 1420
+    },
+    {
+      "epoch": 2.4591573516766982,
+      "grad_norm": 0.19205219312460517,
+      "learning_rate": 1.9027699017662194e-05,
+      "loss": 0.3703,
+      "step": 1430
+    },
+    {
+      "epoch": 2.476354256233878,
+      "grad_norm": 0.18216660996651907,
+      "learning_rate": 1.7868311845627472e-05,
+      "loss": 0.3754,
+      "step": 1440
+    },
+    {
+      "epoch": 2.4935511607910574,
+      "grad_norm": 0.18809064927597854,
+      "learning_rate": 1.6741893527493858e-05,
+      "loss": 0.3707,
+      "step": 1450
+    },
+    {
+      "epoch": 2.5107480653482375,
+      "grad_norm": 0.18562863684234907,
+      "learning_rate": 1.564889622395349e-05,
+      "loss": 0.3698,
+      "step": 1460
+    },
+    {
+      "epoch": 2.527944969905417,
+      "grad_norm": 0.18350802969723742,
+      "learning_rate": 1.4589758680017263e-05,
+      "loss": 0.3733,
+      "step": 1470
+    },
+    {
+      "epoch": 2.5451418744625967,
+      "grad_norm": 0.18684066406776073,
+      "learning_rate": 1.356490604889622e-05,
+      "loss": 0.3787,
+      "step": 1480
+    },
+    {
+      "epoch": 2.5623387790197762,
+      "grad_norm": 0.1811515744478759,
+      "learning_rate": 1.2574749721338874e-05,
+      "loss": 0.3769,
+      "step": 1490
+    },
+    {
+      "epoch": 2.5795356835769563,
+      "grad_norm": 0.18069550077179752,
+      "learning_rate": 1.1619687160492953e-05,
+      "loss": 0.3701,
+      "step": 1500
+    },
+    {
+      "epoch": 2.596732588134136,
+      "grad_norm": 0.18462589607546182,
+      "learning_rate": 1.0700101742357926e-05,
+      "loss": 0.3744,
+      "step": 1510
+    },
+    {
+      "epoch": 2.6139294926913155,
+      "grad_norm": 0.18712041177077796,
+      "learning_rate": 9.816362601892326e-06,
+      "loss": 0.3677,
+      "step": 1520
+    },
+    {
+      "epoch": 2.6311263972484955,
+      "grad_norm": 0.18908434345209518,
+      "learning_rate": 8.968824484837578e-06,
+      "loss": 0.3685,
+      "step": 1530
+    },
+    {
+      "epoch": 2.648323301805675,
+      "grad_norm": 0.1847336144413599,
+      "learning_rate": 8.157827605317892e-06,
+      "loss": 0.3713,
+      "step": 1540
+    },
+    {
+      "epoch": 2.6655202063628547,
+      "grad_norm": 0.18269892602018764,
+      "learning_rate": 7.383697509273424e-06,
+      "loss": 0.367,
+      "step": 1550
+    },
+    {
+      "epoch": 2.6827171109200343,
+      "grad_norm": 0.18647288570365314,
+      "learning_rate": 6.646744943781325e-06,
+      "loss": 0.3724,
+      "step": 1560
+    },
+    {
+      "epoch": 2.699914015477214,
+      "grad_norm": 0.18698726008663819,
+      "learning_rate": 5.947265732317408e-06,
+      "loss": 0.3659,
+      "step": 1570
+    },
+    {
+      "epoch": 2.717110920034394,
+      "grad_norm": 0.18309326292481923,
+      "learning_rate": 5.285540656008303e-06,
+      "loss": 0.3757,
+      "step": 1580
+    },
+    {
+      "epoch": 2.7343078245915735,
+      "grad_norm": 0.18668212507421977,
+      "learning_rate": 4.6618353409217386e-06,
+      "loss": 0.3673,
+      "step": 1590
+    },
+    {
+      "epoch": 2.751504729148753,
+      "grad_norm": 0.1840554864418513,
+      "learning_rate": 4.076400151440485e-06,
+      "loss": 0.367,
+      "step": 1600
+    },
+    {
+      "epoch": 2.768701633705933,
+      "grad_norm": 0.1838274059601279,
+      "learning_rate": 3.529470089762421e-06,
+      "loss": 0.3676,
+      "step": 1610
+    },
+    {
+      "epoch": 2.7858985382631127,
+      "grad_norm": 0.1905340116630113,
+      "learning_rate": 3.021264701567206e-06,
+      "loss": 0.3745,
+      "step": 1620
+    },
+    {
+      "epoch": 2.8030954428202923,
+      "grad_norm": 0.1859751703399611,
+      "learning_rate": 2.551987987887461e-06,
+      "loss": 0.369,
+      "step": 1630
+    },
+    {
+      "epoch": 2.820292347377472,
+      "grad_norm": 0.18905809126144052,
+      "learning_rate": 2.1218283232198212e-06,
+      "loss": 0.3774,
+      "step": 1640
+    },
+    {
+      "epoch": 2.8374892519346515,
+      "grad_norm": 0.1853105742586414,
+      "learning_rate": 1.7309583799086094e-06,
+      "loss": 0.3726,
+      "step": 1650
+    },
+    {
+      "epoch": 2.8546861564918316,
+      "grad_norm": 0.1840942740034349,
+      "learning_rate": 1.3795350588327261e-06,
+      "loss": 0.3779,
+      "step": 1660
+    },
+    {
+      "epoch": 2.871883061049011,
+      "grad_norm": 0.18025938643579378,
+      "learning_rate": 1.0676994264232854e-06,
+      "loss": 0.3692,
+      "step": 1670
+    },
+    {
+      "epoch": 2.8890799656061907,
+      "grad_norm": 0.18685969171459274,
+      "learning_rate": 7.955766580375335e-07,
+      "loss": 0.3688,
+      "step": 1680
+    },
+    {
+      "epoch": 2.9062768701633708,
+      "grad_norm": 0.1830773378552538,
+      "learning_rate": 5.632759877116422e-07,
+      "loss": 0.3738,
+      "step": 1690
+    },
+    {
+      "epoch": 2.9234737747205504,
+      "grad_norm": 0.18687096055320696,
+      "learning_rate": 3.708906643125509e-07,
+      "loss": 0.3754,
+      "step": 1700
+    },
+    {
+      "epoch": 2.94067067927773,
+      "grad_norm": 0.18843515900821306,
+      "learning_rate": 2.184979141065413e-07,
+      "loss": 0.374,
+      "step": 1710
+    },
+    {
+      "epoch": 2.9578675838349096,
+      "grad_norm": 0.1907738099777219,
+      "learning_rate": 1.061589097595017e-07,
+      "loss": 0.3719,
+      "step": 1720
+    },
+    {
+      "epoch": 2.9750644883920896,
+      "grad_norm": 0.18629496289787387,
+      "learning_rate": 3.3918745781291725e-08,
+      "loss": 0.3734,
+      "step": 1730
+    },
+    {
+      "epoch": 2.992261392949269,
+      "grad_norm": 0.1807736649886976,
+      "learning_rate": 1.8064204241774462e-09,
+      "loss": 0.3669,
+      "step": 1740
+    },
+    {
+      "epoch": 2.997420464316423,
+      "step": 1743,
+      "total_flos": 1.7624457120055296e+16,
+      "train_loss": 0.4506890232783245,
+      "train_runtime": 9589.9992,
+      "train_samples_per_second": 5.818,
+      "train_steps_per_second": 0.182
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1743,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.7624457120055296e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec8059564270233c463911936512c79ec62b3b3bf31419a7cf3e709dbd672566
+size 6904