Upload folder using huggingface_hub

README.md

@@ -0,0 +1,62 @@
+---
+base_model: liquidai/LFM2-1.2B
+library_name: peft
+model_name: EventModel-1.2B
+tags:
+- base_model:adapter:liquidai/LFM2-1.2B
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for EventModel-1.2B
+
+This model is a fine-tuned version of [liquidai/LFM2-1.2B](https://huggingface.co/liquidai/LFM2-1.2B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.27.2
+- Transformers: 5.1.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

adapter_config.json

@@ -0,0 +1,47 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "liquidai/LFM2-1.2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "v_proj",
+    "w3",
+    "in_proj",
+    "out_proj",
+    "q_proj",
+    "w2",
+    "w1"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19812c8594095365dc2b43bf18e294981f38b2f3c366474ad5fda67906de336
+size 44457856

chat_template.jinja

@@ -0,0 +1,37 @@
+{{- bos_token -}}
+{%- set system_prompt = "" -%}
+{%- set ns = namespace(system_prompt="") -%}
+{%- if messages[0]["role"] == "system" -%}
+	{%- set ns.system_prompt = messages[0]["content"] -%}
+	{%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+	{%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%}
+	{%- for tool in tools -%}
+		{%- if tool is not string -%}
+            {%- set tool = tool | tojson -%}
+		{%- endif -%}
+		{%- set ns.system_prompt = ns.system_prompt + tool -%}
+        {%- if not loop.last -%}
+            {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+        {%- endif -%}
+	{%- endfor -%}
+	{%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+	{{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+	{{- "<|im_start|>" + message["role"] + "\n" -}}
+	{%- set content = message["content"] -%}
+	{%- if content is not string -%}
+		{%- set content = content | tojson -%}
+	{%- endif -%}
+	{%- if message["role"] == "tool" -%}
+		{%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%}
+	{%- endif -%}
+	{{- content + "<|im_end|>\n" -}}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+	{{- "<|im_start|>assistant\n" -}}
+{%- endif -%}

checkpoint-105/README.md

@@ -0,0 +1,209 @@
+---
+base_model: liquidai/LFM2-1.2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:liquidai/LFM2-1.2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-105/adapter_config.json

@@ -0,0 +1,47 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "liquidai/LFM2-1.2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "v_proj",
+    "w3",
+    "in_proj",
+    "out_proj",
+    "q_proj",
+    "w2",
+    "w1"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-105/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19812c8594095365dc2b43bf18e294981f38b2f3c366474ad5fda67906de336
+size 44457856

checkpoint-105/chat_template.jinja

@@ -0,0 +1,37 @@
+{{- bos_token -}}
+{%- set system_prompt = "" -%}
+{%- set ns = namespace(system_prompt="") -%}
+{%- if messages[0]["role"] == "system" -%}
+	{%- set ns.system_prompt = messages[0]["content"] -%}
+	{%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+	{%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%}
+	{%- for tool in tools -%}
+		{%- if tool is not string -%}
+            {%- set tool = tool | tojson -%}
+		{%- endif -%}
+		{%- set ns.system_prompt = ns.system_prompt + tool -%}
+        {%- if not loop.last -%}
+            {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+        {%- endif -%}
+	{%- endfor -%}
+	{%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+	{{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+	{{- "<|im_start|>" + message["role"] + "\n" -}}
+	{%- set content = message["content"] -%}
+	{%- if content is not string -%}
+		{%- set content = content | tojson -%}
+	{%- endif -%}
+	{%- if message["role"] == "tool" -%}
+		{%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%}
+	{%- endif -%}
+	{{- content + "<|im_end|>\n" -}}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+	{{- "<|im_start|>assistant\n" -}}
+{%- endif -%}

checkpoint-105/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7e6f7734118dffa06cda9f103852560fd2f38c76dbf1e81fce9dd6148b6c55d
+size 22781963

checkpoint-105/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab90f93fa91adbe2cb633343c3677564f71a43ed32e948455d4a727280b4483c
+size 14645

checkpoint-105/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ec32c4320805a3f35a1c5c994350ccf8df6e162d5cff6b8f795489ce7c9ed7f
+size 1465

checkpoint-105/tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "legacy": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

checkpoint-105/trainer_state.json

@@ -0,0 +1,1084 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 105,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.5899229049682617,
+      "epoch": 0.029304029304029304,
+      "grad_norm": 2.8183693885803223,
+      "learning_rate": 0.0002,
+      "loss": 4.0399041175842285,
+      "mean_token_accuracy": 0.2504686303436756,
+      "num_tokens": 943.0,
+      "step": 1
+    },
+    {
+      "entropy": 2.4779607951641083,
+      "epoch": 0.05860805860805861,
+      "grad_norm": 2.0576627254486084,
+      "learning_rate": 0.0001980952380952381,
+      "loss": 3.5690760612487793,
+      "mean_token_accuracy": 0.33353323861956596,
+      "num_tokens": 1943.0,
+      "step": 2
+    },
+    {
+      "entropy": 2.6818120777606964,
+      "epoch": 0.08791208791208792,
+      "grad_norm": 1.8687777519226074,
+      "learning_rate": 0.0001961904761904762,
+      "loss": 3.2856528759002686,
+      "mean_token_accuracy": 0.3809729106724262,
+      "num_tokens": 2888.0,
+      "step": 3
+    },
+    {
+      "entropy": 2.715443432331085,
+      "epoch": 0.11721611721611722,
+      "grad_norm": 1.5808842182159424,
+      "learning_rate": 0.0001942857142857143,
+      "loss": 2.8003592491149902,
+      "mean_token_accuracy": 0.40646324679255486,
+      "num_tokens": 3931.0,
+      "step": 4
+    },
+    {
+      "entropy": 2.8187700510025024,
+      "epoch": 0.14652014652014653,
+      "grad_norm": 1.4385764598846436,
+      "learning_rate": 0.0001923809523809524,
+      "loss": 2.7211196422576904,
+      "mean_token_accuracy": 0.4199763052165508,
+      "num_tokens": 4819.0,
+      "step": 5
+    },
+    {
+      "entropy": 2.79895681142807,
+      "epoch": 0.17582417582417584,
+      "grad_norm": 1.1686122417449951,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 2.6673367023468018,
+      "mean_token_accuracy": 0.4695504158735275,
+      "num_tokens": 5830.0,
+      "step": 6
+    },
+    {
+      "entropy": 2.7103140354156494,
+      "epoch": 0.20512820512820512,
+      "grad_norm": 1.1704784631729126,
+      "learning_rate": 0.00018857142857142857,
+      "loss": 2.409618854522705,
+      "mean_token_accuracy": 0.5037945993244648,
+      "num_tokens": 6723.0,
+      "step": 7
+    },
+    {
+      "entropy": 2.6644202172756195,
+      "epoch": 0.23443223443223443,
+      "grad_norm": 1.143977165222168,
+      "learning_rate": 0.0001866666666666667,
+      "loss": 2.4525094032287598,
+      "mean_token_accuracy": 0.4569362699985504,
+      "num_tokens": 7724.0,
+      "step": 8
+    },
+    {
+      "entropy": 2.5918588638305664,
+      "epoch": 0.26373626373626374,
+      "grad_norm": 1.000954031944275,
+      "learning_rate": 0.00018476190476190478,
+      "loss": 2.396008253097534,
+      "mean_token_accuracy": 0.4624715633690357,
+      "num_tokens": 8706.0,
+      "step": 9
+    },
+    {
+      "entropy": 2.3777745366096497,
+      "epoch": 0.29304029304029305,
+      "grad_norm": 1.1188299655914307,
+      "learning_rate": 0.00018285714285714286,
+      "loss": 2.281681776046753,
+      "mean_token_accuracy": 0.5069036744534969,
+      "num_tokens": 9652.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.403194159269333,
+      "epoch": 0.32234432234432236,
+      "grad_norm": 0.9767391085624695,
+      "learning_rate": 0.00018095238095238095,
+      "loss": 2.354844331741333,
+      "mean_token_accuracy": 0.4832206554710865,
+      "num_tokens": 10647.0,
+      "step": 11
+    },
+    {
+      "entropy": 2.3256518244743347,
+      "epoch": 0.3516483516483517,
+      "grad_norm": 0.9643223285675049,
+      "learning_rate": 0.00017904761904761907,
+      "loss": 2.2260360717773438,
+      "mean_token_accuracy": 0.5200235471129417,
+      "num_tokens": 11706.0,
+      "step": 12
+    },
+    {
+      "entropy": 2.30185005068779,
+      "epoch": 0.38095238095238093,
+      "grad_norm": 1.1241270303726196,
+      "learning_rate": 0.00017714285714285713,
+      "loss": 2.342158317565918,
+      "mean_token_accuracy": 0.49673087149858475,
+      "num_tokens": 12742.0,
+      "step": 13
+    },
+    {
+      "entropy": 2.2283116281032562,
+      "epoch": 0.41025641025641024,
+      "grad_norm": 1.0168424844741821,
+      "learning_rate": 0.00017523809523809525,
+      "loss": 2.3015754222869873,
+      "mean_token_accuracy": 0.5138893984258175,
+      "num_tokens": 13651.0,
+      "step": 14
+    },
+    {
+      "entropy": 2.251826673746109,
+      "epoch": 0.43956043956043955,
+      "grad_norm": 0.9726622700691223,
+      "learning_rate": 0.00017333333333333334,
+      "loss": 2.2128169536590576,
+      "mean_token_accuracy": 0.5023349672555923,
+      "num_tokens": 14713.0,
+      "step": 15
+    },
+    {
+      "entropy": 2.334069699048996,
+      "epoch": 0.46886446886446886,
+      "grad_norm": 0.9483299255371094,
+      "learning_rate": 0.00017142857142857143,
+      "loss": 2.1633079051971436,
+      "mean_token_accuracy": 0.5210121423006058,
+      "num_tokens": 16110.0,
+      "step": 16
+    },
+    {
+      "entropy": 2.236449494957924,
+      "epoch": 0.4981684981684982,
+      "grad_norm": 0.8546555638313293,
+      "learning_rate": 0.00016952380952380954,
+      "loss": 2.108891725540161,
+      "mean_token_accuracy": 0.5399356037378311,
+      "num_tokens": 17232.0,
+      "step": 17
+    },
+    {
+      "entropy": 2.360167473554611,
+      "epoch": 0.5274725274725275,
+      "grad_norm": 0.9520578384399414,
+      "learning_rate": 0.00016761904761904763,
+      "loss": 2.2965035438537598,
+      "mean_token_accuracy": 0.5066869631409645,
+      "num_tokens": 18287.0,
+      "step": 18
+    },
+    {
+      "entropy": 2.2901051938533783,
+      "epoch": 0.5567765567765568,
+      "grad_norm": 0.8338903188705444,
+      "learning_rate": 0.00016571428571428575,
+      "loss": 2.23210072517395,
+      "mean_token_accuracy": 0.5214172080159187,
+      "num_tokens": 19543.0,
+      "step": 19
+    },
+    {
+      "entropy": 2.297758638858795,
+      "epoch": 0.5860805860805861,
+      "grad_norm": 1.0154532194137573,
+      "learning_rate": 0.0001638095238095238,
+      "loss": 2.3786871433258057,
+      "mean_token_accuracy": 0.4790012463927269,
+      "num_tokens": 20517.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.2084394991397858,
+      "epoch": 0.6153846153846154,
+      "grad_norm": 1.0085352659225464,
+      "learning_rate": 0.00016190476190476192,
+      "loss": 2.2479264736175537,
+      "mean_token_accuracy": 0.4972332492470741,
+      "num_tokens": 21420.0,
+      "step": 21
+    },
+    {
+      "entropy": 2.1873691976070404,
+      "epoch": 0.6446886446886447,
+      "grad_norm": 0.8938171863555908,
+      "learning_rate": 0.00016,
+      "loss": 2.1455302238464355,
+      "mean_token_accuracy": 0.5137167796492577,
+      "num_tokens": 22497.0,
+      "step": 22
+    },
+    {
+      "entropy": 2.1501827239990234,
+      "epoch": 0.673992673992674,
+      "grad_norm": 0.9702470898628235,
+      "learning_rate": 0.0001580952380952381,
+      "loss": 2.364546775817871,
+      "mean_token_accuracy": 0.4991302490234375,
+      "num_tokens": 23649.0,
+      "step": 23
+    },
+    {
+      "entropy": 2.1364653259515762,
+      "epoch": 0.7032967032967034,
+      "grad_norm": 0.9650313258171082,
+      "learning_rate": 0.0001561904761904762,
+      "loss": 2.18817400932312,
+      "mean_token_accuracy": 0.5010919496417046,
+      "num_tokens": 24801.0,
+      "step": 24
+    },
+    {
+      "entropy": 2.0752783119678497,
+      "epoch": 0.7326007326007326,
+      "grad_norm": 0.9651536345481873,
+      "learning_rate": 0.0001542857142857143,
+      "loss": 2.2106223106384277,
+      "mean_token_accuracy": 0.5068995915353298,
+      "num_tokens": 25689.0,
+      "step": 25
+    },
+    {
+      "entropy": 2.172010287642479,
+      "epoch": 0.7619047619047619,
+      "grad_norm": 0.897559642791748,
+      "learning_rate": 0.00015238095238095237,
+      "loss": 2.275151014328003,
+      "mean_token_accuracy": 0.5263335853815079,
+      "num_tokens": 26667.0,
+      "step": 26
+    },
+    {
+      "entropy": 2.2060853838920593,
+      "epoch": 0.7912087912087912,
+      "grad_norm": 0.9621699452400208,
+      "learning_rate": 0.00015047619047619048,
+      "loss": 2.229595899581909,
+      "mean_token_accuracy": 0.5082523375749588,
+      "num_tokens": 27656.0,
+      "step": 27
+    },
+    {
+      "entropy": 2.24511581659317,
+      "epoch": 0.8205128205128205,
+      "grad_norm": 0.8171648979187012,
+      "learning_rate": 0.00014857142857142857,
+      "loss": 2.2346324920654297,
+      "mean_token_accuracy": 0.5153078399598598,
+      "num_tokens": 28821.0,
+      "step": 28
+    },
+    {
+      "entropy": 2.2484240978956223,
+      "epoch": 0.8498168498168498,
+      "grad_norm": 0.8533217906951904,
+      "learning_rate": 0.00014666666666666666,
+      "loss": 2.2458832263946533,
+      "mean_token_accuracy": 0.5125137269496918,
+      "num_tokens": 30047.0,
+      "step": 29
+    },
+    {
+      "entropy": 2.2615038454532623,
+      "epoch": 0.8791208791208791,
+      "grad_norm": 0.9642089009284973,
+      "learning_rate": 0.00014476190476190475,
+      "loss": 2.219306468963623,
+      "mean_token_accuracy": 0.520592100918293,
+      "num_tokens": 31006.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.2009344547986984,
+      "epoch": 0.9084249084249084,
+      "grad_norm": 0.9399288296699524,
+      "learning_rate": 0.00014285714285714287,
+      "loss": 2.069526433944702,
+      "mean_token_accuracy": 0.5263588838279247,
+      "num_tokens": 32025.0,
+      "step": 31
+    },
+    {
+      "entropy": 2.1680870354175568,
+      "epoch": 0.9377289377289377,
+      "grad_norm": 0.8993312120437622,
+      "learning_rate": 0.00014095238095238096,
+      "loss": 2.200000524520874,
+      "mean_token_accuracy": 0.5152596533298492,
+      "num_tokens": 32906.0,
+      "step": 32
+    },
+    {
+      "entropy": 2.10675248503685,
+      "epoch": 0.967032967032967,
+      "grad_norm": 0.8099379539489746,
+      "learning_rate": 0.00013904761904761905,
+      "loss": 2.013751268386841,
+      "mean_token_accuracy": 0.5571991391479969,
+      "num_tokens": 33932.0,
+      "step": 33
+    },
+    {
+      "entropy": 2.1444759368896484,
+      "epoch": 0.9963369963369964,
+      "grad_norm": 0.8524768352508545,
+      "learning_rate": 0.00013714285714285716,
+      "loss": 2.191409111022949,
+      "mean_token_accuracy": 0.5195218212902546,
+      "num_tokens": 34923.0,
+      "step": 34
+    },
+    {
+      "entropy": 2.206982374191284,
+      "epoch": 1.0,
+      "grad_norm": 2.50785493850708,
+      "learning_rate": 0.00013523809523809525,
+      "loss": 2.276951551437378,
+      "mean_token_accuracy": 0.5,
+      "num_tokens": 35019.0,
+      "step": 35
+    },
+    {
+      "entropy": 2.1279809772968292,
+      "epoch": 1.0293040293040292,
+      "grad_norm": 0.812450647354126,
+      "learning_rate": 0.00013333333333333334,
+      "loss": 2.06909441947937,
+      "mean_token_accuracy": 0.5160608440637589,
+      "num_tokens": 36132.0,
+      "step": 36
+    },
+    {
+      "entropy": 2.0765769630670547,
+      "epoch": 1.0586080586080586,
+      "grad_norm": 0.8679144382476807,
+      "learning_rate": 0.00013142857142857143,
+      "loss": 2.101022720336914,
+      "mean_token_accuracy": 0.5235091745853424,
+      "num_tokens": 37176.0,
+      "step": 37
+    },
+    {
+      "entropy": 2.063512533903122,
+      "epoch": 1.0879120879120878,
+      "grad_norm": 1.0040966272354126,
+      "learning_rate": 0.00012952380952380954,
+      "loss": 2.1887731552124023,
+      "mean_token_accuracy": 0.5090154893696308,
+      "num_tokens": 38125.0,
+      "step": 38
+    },
+    {
+      "entropy": 1.9415698498487473,
+      "epoch": 1.1172161172161172,
+      "grad_norm": 0.8333518505096436,
+      "learning_rate": 0.0001276190476190476,
+      "loss": 1.8331354856491089,
+      "mean_token_accuracy": 0.5899114012718201,
+      "num_tokens": 39253.0,
+      "step": 39
+    },
+    {
+      "entropy": 2.115827590227127,
+      "epoch": 1.1465201465201464,
+      "grad_norm": 0.8999753594398499,
+      "learning_rate": 0.00012571428571428572,
+      "loss": 2.0922064781188965,
+      "mean_token_accuracy": 0.5041939504444599,
+      "num_tokens": 40456.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.0162982046604156,
+      "epoch": 1.1758241758241759,
+      "grad_norm": 0.889089822769165,
+      "learning_rate": 0.0001238095238095238,
+      "loss": 1.9345817565917969,
+      "mean_token_accuracy": 0.5448144525289536,
+      "num_tokens": 41595.0,
+      "step": 41
+    },
+    {
+      "entropy": 1.9902137368917465,
+      "epoch": 1.205128205128205,
+      "grad_norm": 0.9763742685317993,
+      "learning_rate": 0.00012190476190476193,
+      "loss": 1.966817855834961,
+      "mean_token_accuracy": 0.5384696908295155,
+      "num_tokens": 42494.0,
+      "step": 42
+    },
+    {
+      "entropy": 1.9661571383476257,
+      "epoch": 1.2344322344322345,
+      "grad_norm": 0.9287217259407043,
+      "learning_rate": 0.00012,
+      "loss": 1.9149142503738403,
+      "mean_token_accuracy": 0.5685045793652534,
+      "num_tokens": 43418.0,
+      "step": 43
+    },
+    {
+      "entropy": 1.8985900431871414,
+      "epoch": 1.2637362637362637,
+      "grad_norm": 0.9483775496482849,
+      "learning_rate": 0.0001180952380952381,
+      "loss": 1.9076926708221436,
+      "mean_token_accuracy": 0.5475717112421989,
+      "num_tokens": 44423.0,
+      "step": 44
+    },
+    {
+      "entropy": 1.9549081772565842,
+      "epoch": 1.293040293040293,
+      "grad_norm": 0.9189373254776001,
+      "learning_rate": 0.00011619047619047621,
+      "loss": 1.9420613050460815,
+      "mean_token_accuracy": 0.5473757795989513,
+      "num_tokens": 45500.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.891863077878952,
+      "epoch": 1.3223443223443223,
+      "grad_norm": 0.9721280932426453,
+      "learning_rate": 0.00011428571428571428,
+      "loss": 1.9221911430358887,
+      "mean_token_accuracy": 0.5582568794488907,
+      "num_tokens": 46546.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.9727875739336014,
+      "epoch": 1.3516483516483517,
+      "grad_norm": 1.0340529680252075,
+      "learning_rate": 0.00011238095238095239,
+      "loss": 1.953711748123169,
+      "mean_token_accuracy": 0.5584920011460781,
+      "num_tokens": 47507.0,
+      "step": 47
+    },
+    {
+      "entropy": 1.8502823263406754,
+      "epoch": 1.380952380952381,
+      "grad_norm": 0.9082620143890381,
+      "learning_rate": 0.00011047619047619049,
+      "loss": 1.883126139640808,
+      "mean_token_accuracy": 0.5711723491549492,
+      "num_tokens": 48476.0,
+      "step": 48
+    },
+    {
+      "entropy": 2.0381594747304916,
+      "epoch": 1.4102564102564101,
+      "grad_norm": 1.0055015087127686,
+      "learning_rate": 0.00010857142857142856,
+      "loss": 2.026916027069092,
+      "mean_token_accuracy": 0.548699252307415,
+      "num_tokens": 49440.0,
+      "step": 49
+    },
+    {
+      "entropy": 1.9274516850709915,
+      "epoch": 1.4395604395604396,
+      "grad_norm": 0.9569126963615417,
+      "learning_rate": 0.00010666666666666667,
+      "loss": 1.9325764179229736,
+      "mean_token_accuracy": 0.5698802806437016,
+      "num_tokens": 50392.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.097727954387665,
+      "epoch": 1.468864468864469,
+      "grad_norm": 0.9394485354423523,
+      "learning_rate": 0.00010476190476190477,
+      "loss": 2.034787654876709,
+      "mean_token_accuracy": 0.5258443988859653,
+      "num_tokens": 51494.0,
+      "step": 51
+    },
+    {
+      "entropy": 1.9933644086122513,
+      "epoch": 1.4981684981684982,
+      "grad_norm": 1.0003395080566406,
+      "learning_rate": 0.00010285714285714286,
+      "loss": 1.834027886390686,
+      "mean_token_accuracy": 0.5819921940565109,
+      "num_tokens": 52453.0,
+      "step": 52
+    },
+    {
+      "entropy": 2.053115800023079,
+      "epoch": 1.5274725274725274,
+      "grad_norm": 1.0380909442901611,
+      "learning_rate": 0.00010095238095238096,
+      "loss": 2.0844743251800537,
+      "mean_token_accuracy": 0.5501821637153625,
+      "num_tokens": 53432.0,
+      "step": 53
+    },
+    {
+      "entropy": 1.8703102469444275,
+      "epoch": 1.5567765567765568,
+      "grad_norm": 1.0789860486984253,
+      "learning_rate": 9.904761904761905e-05,
+      "loss": 1.8772366046905518,
+      "mean_token_accuracy": 0.550880528986454,
+      "num_tokens": 54359.0,
+      "step": 54
+    },
+    {
+      "entropy": 2.006308153271675,
+      "epoch": 1.5860805860805862,
+      "grad_norm": 1.039587140083313,
+      "learning_rate": 9.714285714285715e-05,
+      "loss": 1.9366910457611084,
+      "mean_token_accuracy": 0.5642399489879608,
+      "num_tokens": 55283.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.9818164855241776,
+      "epoch": 1.6153846153846154,
+      "grad_norm": 1.0115448236465454,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 1.959208607673645,
+      "mean_token_accuracy": 0.5404646620154381,
+      "num_tokens": 56360.0,
+      "step": 56
+    },
+    {
+      "entropy": 1.9919450134038925,
+      "epoch": 1.6446886446886446,
+      "grad_norm": 1.0340183973312378,
+      "learning_rate": 9.333333333333334e-05,
+      "loss": 1.8634270429611206,
+      "mean_token_accuracy": 0.563919834792614,
+      "num_tokens": 57270.0,
+      "step": 57
+    },
+    {
+      "entropy": 2.045059621334076,
+      "epoch": 1.673992673992674,
+      "grad_norm": 1.0698320865631104,
+      "learning_rate": 9.142857142857143e-05,
+      "loss": 1.9465467929840088,
+      "mean_token_accuracy": 0.549336139112711,
+      "num_tokens": 58183.0,
+      "step": 58
+    },
+    {
+      "entropy": 2.011963799595833,
+      "epoch": 1.7032967032967035,
+      "grad_norm": 1.0501713752746582,
+      "learning_rate": 8.952380952380953e-05,
+      "loss": 2.0452449321746826,
+      "mean_token_accuracy": 0.5631852373480797,
+      "num_tokens": 59179.0,
+      "step": 59
+    },
+    {
+      "entropy": 1.9702570289373398,
+      "epoch": 1.7326007326007327,
+      "grad_norm": 1.0021151304244995,
+      "learning_rate": 8.761904761904762e-05,
+      "loss": 1.8943660259246826,
+      "mean_token_accuracy": 0.545048289000988,
+      "num_tokens": 60384.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.8890659809112549,
+      "epoch": 1.7619047619047619,
+      "grad_norm": 1.0946227312088013,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 1.9704440832138062,
+      "mean_token_accuracy": 0.569721557199955,
+      "num_tokens": 61392.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.9658374786376953,
+      "epoch": 1.791208791208791,
+      "grad_norm": 1.072704553604126,
+      "learning_rate": 8.380952380952382e-05,
+      "loss": 1.9025211334228516,
+      "mean_token_accuracy": 0.5500420220196247,
+      "num_tokens": 62541.0,
+      "step": 62
+    },
+    {
+      "entropy": 1.9273017197847366,
+      "epoch": 1.8205128205128205,
+      "grad_norm": 1.071745753288269,
+      "learning_rate": 8.19047619047619e-05,
+      "loss": 1.871708869934082,
+      "mean_token_accuracy": 0.5801190733909607,
+      "num_tokens": 63607.0,
+      "step": 63
+    },
+    {
+      "entropy": 2.1368707567453384,
+      "epoch": 1.84981684981685,
+      "grad_norm": 1.0277183055877686,
+      "learning_rate": 8e-05,
+      "loss": 2.0618956089019775,
+      "mean_token_accuracy": 0.5408336892724037,
+      "num_tokens": 64724.0,
+      "step": 64
+    },
+    {
+      "entropy": 1.951027974486351,
+      "epoch": 1.879120879120879,
+      "grad_norm": 1.0923572778701782,
+      "learning_rate": 7.80952380952381e-05,
+      "loss": 1.9516839981079102,
+      "mean_token_accuracy": 0.5601083487272263,
+      "num_tokens": 65691.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.937852904200554,
+      "epoch": 1.9084249084249083,
+      "grad_norm": 1.1021313667297363,
+      "learning_rate": 7.619047619047618e-05,
+      "loss": 1.9671883583068848,
+      "mean_token_accuracy": 0.5423109643161297,
+      "num_tokens": 66741.0,
+      "step": 66
+    },
+    {
+      "entropy": 1.9332957565784454,
+      "epoch": 1.9377289377289377,
+      "grad_norm": 0.963511049747467,
+      "learning_rate": 7.428571428571429e-05,
+      "loss": 1.9559173583984375,
+      "mean_token_accuracy": 0.5629658326506615,
+      "num_tokens": 67942.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.9278145879507065,
+      "epoch": 1.9670329670329672,
+      "grad_norm": 1.0941733121871948,
+      "learning_rate": 7.238095238095238e-05,
+      "loss": 1.785203456878662,
+      "mean_token_accuracy": 0.5560223236680031,
+      "num_tokens": 68867.0,
+      "step": 68
+    },
+    {
+      "entropy": 1.9755759537220001,
+      "epoch": 1.9963369963369964,
+      "grad_norm": 1.2056344747543335,
+      "learning_rate": 7.047619047619048e-05,
+      "loss": 2.001420497894287,
+      "mean_token_accuracy": 0.5633600018918514,
+      "num_tokens": 69898.0,
+      "step": 69
+    },
+    {
+      "entropy": 1.8418163061141968,
+      "epoch": 2.0,
+      "grad_norm": 2.6260459423065186,
+      "learning_rate": 6.857142857142858e-05,
+      "loss": 2.0277798175811768,
+      "mean_token_accuracy": 0.5652173757553101,
+      "num_tokens": 70038.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.8679307699203491,
+      "epoch": 2.029304029304029,
+      "grad_norm": 1.097974181175232,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 1.7054555416107178,
+      "mean_token_accuracy": 0.6016205847263336,
+      "num_tokens": 70963.0,
+      "step": 71
+    },
+    {
+      "entropy": 1.9588469415903091,
+      "epoch": 2.0586080586080584,
+      "grad_norm": 1.1219500303268433,
+      "learning_rate": 6.476190476190477e-05,
+      "loss": 1.9303096532821655,
+      "mean_token_accuracy": 0.5386716388165951,
+      "num_tokens": 71923.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.85378035902977,
+      "epoch": 2.087912087912088,
+      "grad_norm": 1.031679630279541,
+      "learning_rate": 6.285714285714286e-05,
+      "loss": 1.6135412454605103,
+      "mean_token_accuracy": 0.6041380316019058,
+      "num_tokens": 73025.0,
+      "step": 73
+    },
+    {
+      "entropy": 1.8411481231451035,
+      "epoch": 2.1172161172161172,
+      "grad_norm": 0.9993085265159607,
+      "learning_rate": 6.0952380952380964e-05,
+      "loss": 1.7551276683807373,
+      "mean_token_accuracy": 0.6028101369738579,
+      "num_tokens": 74212.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.8153350502252579,
+      "epoch": 2.1465201465201464,
+      "grad_norm": 1.08579683303833,
+      "learning_rate": 5.904761904761905e-05,
+      "loss": 1.7711514234542847,
+      "mean_token_accuracy": 0.5654070526361465,
+      "num_tokens": 75290.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.7724721282720566,
+      "epoch": 2.1758241758241756,
+      "grad_norm": 1.1582475900650024,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 1.758057713508606,
+      "mean_token_accuracy": 0.571387343108654,
+      "num_tokens": 76297.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.8428474068641663,
+      "epoch": 2.2051282051282053,
+      "grad_norm": 1.0935100317001343,
+      "learning_rate": 5.5238095238095244e-05,
+      "loss": 1.778435230255127,
+      "mean_token_accuracy": 0.5798092931509018,
+      "num_tokens": 77241.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.7926504015922546,
+      "epoch": 2.2344322344322345,
+      "grad_norm": 1.083114743232727,
+      "learning_rate": 5.333333333333333e-05,
+      "loss": 1.6600806713104248,
+      "mean_token_accuracy": 0.6051888987421989,
+      "num_tokens": 78235.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.9213951528072357,
+      "epoch": 2.2637362637362637,
+      "grad_norm": 1.1077088117599487,
+      "learning_rate": 5.142857142857143e-05,
+      "loss": 1.8460519313812256,
+      "mean_token_accuracy": 0.5700710937380791,
+      "num_tokens": 79301.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.8596570938825607,
+      "epoch": 2.293040293040293,
+      "grad_norm": 1.2677751779556274,
+      "learning_rate": 4.9523809523809525e-05,
+      "loss": 1.838474154472351,
+      "mean_token_accuracy": 0.5711517967283726,
+      "num_tokens": 80261.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.8317839354276657,
+      "epoch": 2.3223443223443225,
+      "grad_norm": 1.2318886518478394,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 1.7830660343170166,
+      "mean_token_accuracy": 0.5885601341724396,
+      "num_tokens": 81291.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.8027765452861786,
+      "epoch": 2.3516483516483517,
+      "grad_norm": 1.182531476020813,
+      "learning_rate": 4.5714285714285716e-05,
+      "loss": 1.6655972003936768,
+      "mean_token_accuracy": 0.6000265926122665,
+      "num_tokens": 82312.0,
+      "step": 82
+    },
+    {
+      "entropy": 1.8573322594165802,
+      "epoch": 2.380952380952381,
+      "grad_norm": 1.2204681634902954,
+      "learning_rate": 4.380952380952381e-05,
+      "loss": 1.7845535278320312,
+      "mean_token_accuracy": 0.5596524104475975,
+      "num_tokens": 83360.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.857906550168991,
+      "epoch": 2.41025641025641,
+      "grad_norm": 1.2490062713623047,
+      "learning_rate": 4.190476190476191e-05,
+      "loss": 1.8924627304077148,
+      "mean_token_accuracy": 0.5911798849701881,
+      "num_tokens": 84332.0,
+      "step": 84
+    },
+    {
+      "entropy": 1.8203676491975784,
+      "epoch": 2.4395604395604398,
+      "grad_norm": 1.199583649635315,
+      "learning_rate": 4e-05,
+      "loss": 1.7438830137252808,
+      "mean_token_accuracy": 0.5948037281632423,
+      "num_tokens": 85397.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.80222387611866,
+      "epoch": 2.468864468864469,
+      "grad_norm": 1.2759959697723389,
+      "learning_rate": 3.809523809523809e-05,
+      "loss": 1.8114933967590332,
+      "mean_token_accuracy": 0.5650865994393826,
+      "num_tokens": 86477.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.8226999640464783,
+      "epoch": 2.498168498168498,
+      "grad_norm": 1.3133738040924072,
+      "learning_rate": 3.619047619047619e-05,
+      "loss": 1.74210786819458,
+      "mean_token_accuracy": 0.5837778970599174,
+      "num_tokens": 87406.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.767728939652443,
+      "epoch": 2.5274725274725274,
+      "grad_norm": 1.2119284868240356,
+      "learning_rate": 3.428571428571429e-05,
+      "loss": 1.7740098237991333,
+      "mean_token_accuracy": 0.581955574452877,
+      "num_tokens": 88430.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.8365075141191483,
+      "epoch": 2.5567765567765566,
+      "grad_norm": 1.185829758644104,
+      "learning_rate": 3.2380952380952386e-05,
+      "loss": 1.789137601852417,
+      "mean_token_accuracy": 0.5769138634204865,
+      "num_tokens": 89576.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.7178055942058563,
+      "epoch": 2.586080586080586,
+      "grad_norm": 1.2955212593078613,
+      "learning_rate": 3.0476190476190482e-05,
+      "loss": 1.6795040369033813,
+      "mean_token_accuracy": 0.607390895485878,
+      "num_tokens": 90546.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.8126763105392456,
+      "epoch": 2.6153846153846154,
+      "grad_norm": 1.3496512174606323,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 1.850915551185608,
+      "mean_token_accuracy": 0.5883866176009178,
+      "num_tokens": 91545.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.8598168343305588,
+      "epoch": 2.6446886446886446,
+      "grad_norm": 1.3412878513336182,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 1.7600839138031006,
+      "mean_token_accuracy": 0.5819656699895859,
+      "num_tokens": 92511.0,
+      "step": 92
+    },
+    {
+      "entropy": 1.774698480963707,
+      "epoch": 2.6739926739926743,
+      "grad_norm": 1.2525370121002197,
+      "learning_rate": 2.4761904761904762e-05,
+      "loss": 1.6181243658065796,
+      "mean_token_accuracy": 0.5922166630625725,
+      "num_tokens": 93532.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.8692089915275574,
+      "epoch": 2.7032967032967035,
+      "grad_norm": 1.2171218395233154,
+      "learning_rate": 2.2857142857142858e-05,
+      "loss": 1.7427890300750732,
+      "mean_token_accuracy": 0.5718274712562561,
+      "num_tokens": 94605.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.7965329438447952,
+      "epoch": 2.7326007326007327,
+      "grad_norm": 1.164866328239441,
+      "learning_rate": 2.0952380952380954e-05,
+      "loss": 1.6461527347564697,
+      "mean_token_accuracy": 0.601178340613842,
+      "num_tokens": 95721.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.9033904373645782,
+      "epoch": 2.761904761904762,
+      "grad_norm": 1.3114417791366577,
+      "learning_rate": 1.9047619047619046e-05,
+      "loss": 1.7550486326217651,
+      "mean_token_accuracy": 0.5800935998558998,
+      "num_tokens": 96756.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.8053234219551086,
+      "epoch": 2.791208791208791,
+      "grad_norm": 1.3193440437316895,
+      "learning_rate": 1.7142857142857145e-05,
+      "loss": 1.8313217163085938,
+      "mean_token_accuracy": 0.5872214511036873,
+      "num_tokens": 97813.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.8128690719604492,
+      "epoch": 2.8205128205128203,
+      "grad_norm": 1.374079704284668,
+      "learning_rate": 1.5238095238095241e-05,
+      "loss": 1.7221006155014038,
+      "mean_token_accuracy": 0.5857822969555855,
+      "num_tokens": 98716.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.7651506215333939,
+      "epoch": 2.84981684981685,
+      "grad_norm": 1.2412004470825195,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 1.7561471462249756,
+      "mean_token_accuracy": 0.5878495573997498,
+      "num_tokens": 99802.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.8337409943342209,
+      "epoch": 2.879120879120879,
+      "grad_norm": 1.2813513278961182,
+      "learning_rate": 1.1428571428571429e-05,
+      "loss": 1.7412132024765015,
+      "mean_token_accuracy": 0.5852590911090374,
+      "num_tokens": 100832.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.8867168575525284,
+      "epoch": 2.9084249084249083,
+      "grad_norm": 1.2824348211288452,
+      "learning_rate": 9.523809523809523e-06,
+      "loss": 1.8129472732543945,
+      "mean_token_accuracy": 0.5931295305490494,
+      "num_tokens": 101873.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.8725313246250153,
+      "epoch": 2.937728937728938,
+      "grad_norm": 1.3829959630966187,
+      "learning_rate": 7.6190476190476205e-06,
+      "loss": 1.8355034589767456,
+      "mean_token_accuracy": 0.5734075754880905,
+      "num_tokens": 102812.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.6948421895503998,
+      "epoch": 2.967032967032967,
+      "grad_norm": 1.2167892456054688,
+      "learning_rate": 5.7142857142857145e-06,
+      "loss": 1.6377742290496826,
+      "mean_token_accuracy": 0.6147312298417091,
+      "num_tokens": 103878.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.8070212751626968,
+      "epoch": 2.9963369963369964,
+      "grad_norm": 1.238746166229248,
+      "learning_rate": 3.8095238095238102e-06,
+      "loss": 1.7994213104248047,
+      "mean_token_accuracy": 0.5780457705259323,
+      "num_tokens": 104930.0,
+      "step": 104
+    },
+    {
+      "entropy": 1.771175742149353,
+      "epoch": 3.0,
+      "grad_norm": 3.63390851020813,
+      "learning_rate": 1.9047619047619051e-06,
+      "loss": 1.9229103326797485,
+      "mean_token_accuracy": 0.6159999966621399,
+      "num_tokens": 105057.0,
+      "step": 105
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 105,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 782834668394496.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-105/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ba7412e8bcc04ce1aaecd5a483524507e2aac83863d87ce0c7eb227cd9b2bb
+size 5585

checkpoint-35/README.md

@@ -0,0 +1,209 @@
+---
+base_model: liquidai/LFM2-1.2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:liquidai/LFM2-1.2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-35/adapter_config.json

@@ -0,0 +1,47 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "liquidai/LFM2-1.2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "v_proj",
+    "w3",
+    "in_proj",
+    "out_proj",
+    "q_proj",
+    "w2",
+    "w1"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-35/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428f145f705799ec03dd76339887147fa30bd7294cc0b8aeb81a485f80e1b91d
+size 44457856

checkpoint-35/chat_template.jinja

@@ -0,0 +1,37 @@
+{{- bos_token -}}
+{%- set system_prompt = "" -%}
+{%- set ns = namespace(system_prompt="") -%}
+{%- if messages[0]["role"] == "system" -%}
+	{%- set ns.system_prompt = messages[0]["content"] -%}
+	{%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+	{%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%}
+	{%- for tool in tools -%}
+		{%- if tool is not string -%}
+            {%- set tool = tool | tojson -%}
+		{%- endif -%}
+		{%- set ns.system_prompt = ns.system_prompt + tool -%}
+        {%- if not loop.last -%}
+            {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+        {%- endif -%}
+	{%- endfor -%}
+	{%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+	{{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+	{{- "<|im_start|>" + message["role"] + "\n" -}}
+	{%- set content = message["content"] -%}
+	{%- if content is not string -%}
+		{%- set content = content | tojson -%}
+	{%- endif -%}
+	{%- if message["role"] == "tool" -%}
+		{%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%}
+	{%- endif -%}
+	{{- content + "<|im_end|>\n" -}}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+	{{- "<|im_start|>assistant\n" -}}
+{%- endif -%}

checkpoint-35/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f582e3a0da4a30ca4c038be9f7c0113eb75ad0168ae4e3d781bea6d29dd80aae
+size 22781963

checkpoint-35/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d0f6eb329c0a95a4ae3a05d884f9c2990834366a08b9e30a30949b4ece96e18
+size 14645

checkpoint-35/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eb4b45d579fee7e7b473b418068b26f82cdc6768ed7f182ef59ed0402defd17
+size 1465

checkpoint-35/tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "legacy": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

checkpoint-35/trainer_state.json

@@ -0,0 +1,384 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 35,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.5899229049682617,
+      "epoch": 0.029304029304029304,
+      "grad_norm": 2.8183693885803223,
+      "learning_rate": 0.0002,
+      "loss": 4.0399041175842285,
+      "mean_token_accuracy": 0.2504686303436756,
+      "num_tokens": 943.0,
+      "step": 1
+    },
+    {
+      "entropy": 2.4779607951641083,
+      "epoch": 0.05860805860805861,
+      "grad_norm": 2.0576627254486084,
+      "learning_rate": 0.0001980952380952381,
+      "loss": 3.5690760612487793,
+      "mean_token_accuracy": 0.33353323861956596,
+      "num_tokens": 1943.0,
+      "step": 2
+    },
+    {
+      "entropy": 2.6818120777606964,
+      "epoch": 0.08791208791208792,
+      "grad_norm": 1.8687777519226074,
+      "learning_rate": 0.0001961904761904762,
+      "loss": 3.2856528759002686,
+      "mean_token_accuracy": 0.3809729106724262,
+      "num_tokens": 2888.0,
+      "step": 3
+    },
+    {
+      "entropy": 2.715443432331085,
+      "epoch": 0.11721611721611722,
+      "grad_norm": 1.5808842182159424,
+      "learning_rate": 0.0001942857142857143,
+      "loss": 2.8003592491149902,
+      "mean_token_accuracy": 0.40646324679255486,
+      "num_tokens": 3931.0,
+      "step": 4
+    },
+    {
+      "entropy": 2.8187700510025024,
+      "epoch": 0.14652014652014653,
+      "grad_norm": 1.4385764598846436,
+      "learning_rate": 0.0001923809523809524,
+      "loss": 2.7211196422576904,
+      "mean_token_accuracy": 0.4199763052165508,
+      "num_tokens": 4819.0,
+      "step": 5
+    },
+    {
+      "entropy": 2.79895681142807,
+      "epoch": 0.17582417582417584,
+      "grad_norm": 1.1686122417449951,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 2.6673367023468018,
+      "mean_token_accuracy": 0.4695504158735275,
+      "num_tokens": 5830.0,
+      "step": 6
+    },
+    {
+      "entropy": 2.7103140354156494,
+      "epoch": 0.20512820512820512,
+      "grad_norm": 1.1704784631729126,
+      "learning_rate": 0.00018857142857142857,
+      "loss": 2.409618854522705,
+      "mean_token_accuracy": 0.5037945993244648,
+      "num_tokens": 6723.0,
+      "step": 7
+    },
+    {
+      "entropy": 2.6644202172756195,
+      "epoch": 0.23443223443223443,
+      "grad_norm": 1.143977165222168,
+      "learning_rate": 0.0001866666666666667,
+      "loss": 2.4525094032287598,
+      "mean_token_accuracy": 0.4569362699985504,
+      "num_tokens": 7724.0,
+      "step": 8
+    },
+    {
+      "entropy": 2.5918588638305664,
+      "epoch": 0.26373626373626374,
+      "grad_norm": 1.000954031944275,
+      "learning_rate": 0.00018476190476190478,
+      "loss": 2.396008253097534,
+      "mean_token_accuracy": 0.4624715633690357,
+      "num_tokens": 8706.0,
+      "step": 9
+    },
+    {
+      "entropy": 2.3777745366096497,
+      "epoch": 0.29304029304029305,
+      "grad_norm": 1.1188299655914307,
+      "learning_rate": 0.00018285714285714286,
+      "loss": 2.281681776046753,
+      "mean_token_accuracy": 0.5069036744534969,
+      "num_tokens": 9652.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.403194159269333,
+      "epoch": 0.32234432234432236,
+      "grad_norm": 0.9767391085624695,
+      "learning_rate": 0.00018095238095238095,
+      "loss": 2.354844331741333,
+      "mean_token_accuracy": 0.4832206554710865,
+      "num_tokens": 10647.0,
+      "step": 11
+    },
+    {
+      "entropy": 2.3256518244743347,
+      "epoch": 0.3516483516483517,
+      "grad_norm": 0.9643223285675049,
+      "learning_rate": 0.00017904761904761907,
+      "loss": 2.2260360717773438,
+      "mean_token_accuracy": 0.5200235471129417,
+      "num_tokens": 11706.0,
+      "step": 12
+    },
+    {
+      "entropy": 2.30185005068779,
+      "epoch": 0.38095238095238093,
+      "grad_norm": 1.1241270303726196,
+      "learning_rate": 0.00017714285714285713,
+      "loss": 2.342158317565918,
+      "mean_token_accuracy": 0.49673087149858475,
+      "num_tokens": 12742.0,
+      "step": 13
+    },
+    {
+      "entropy": 2.2283116281032562,
+      "epoch": 0.41025641025641024,
+      "grad_norm": 1.0168424844741821,
+      "learning_rate": 0.00017523809523809525,
+      "loss": 2.3015754222869873,
+      "mean_token_accuracy": 0.5138893984258175,
+      "num_tokens": 13651.0,
+      "step": 14
+    },
+    {
+      "entropy": 2.251826673746109,
+      "epoch": 0.43956043956043955,
+      "grad_norm": 0.9726622700691223,
+      "learning_rate": 0.00017333333333333334,
+      "loss": 2.2128169536590576,
+      "mean_token_accuracy": 0.5023349672555923,
+      "num_tokens": 14713.0,
+      "step": 15
+    },
+    {
+      "entropy": 2.334069699048996,
+      "epoch": 0.46886446886446886,
+      "grad_norm": 0.9483299255371094,
+      "learning_rate": 0.00017142857142857143,
+      "loss": 2.1633079051971436,
+      "mean_token_accuracy": 0.5210121423006058,
+      "num_tokens": 16110.0,
+      "step": 16
+    },
+    {
+      "entropy": 2.236449494957924,
+      "epoch": 0.4981684981684982,
+      "grad_norm": 0.8546555638313293,
+      "learning_rate": 0.00016952380952380954,
+      "loss": 2.108891725540161,
+      "mean_token_accuracy": 0.5399356037378311,
+      "num_tokens": 17232.0,
+      "step": 17
+    },
+    {
+      "entropy": 2.360167473554611,
+      "epoch": 0.5274725274725275,
+      "grad_norm": 0.9520578384399414,
+      "learning_rate": 0.00016761904761904763,
+      "loss": 2.2965035438537598,
+      "mean_token_accuracy": 0.5066869631409645,
+      "num_tokens": 18287.0,
+      "step": 18
+    },
+    {
+      "entropy": 2.2901051938533783,
+      "epoch": 0.5567765567765568,
+      "grad_norm": 0.8338903188705444,
+      "learning_rate": 0.00016571428571428575,
+      "loss": 2.23210072517395,
+      "mean_token_accuracy": 0.5214172080159187,
+      "num_tokens": 19543.0,
+      "step": 19
+    },
+    {
+      "entropy": 2.297758638858795,
+      "epoch": 0.5860805860805861,
+      "grad_norm": 1.0154532194137573,
+      "learning_rate": 0.0001638095238095238,
+      "loss": 2.3786871433258057,
+      "mean_token_accuracy": 0.4790012463927269,
+      "num_tokens": 20517.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.2084394991397858,
+      "epoch": 0.6153846153846154,
+      "grad_norm": 1.0085352659225464,
+      "learning_rate": 0.00016190476190476192,
+      "loss": 2.2479264736175537,
+      "mean_token_accuracy": 0.4972332492470741,
+      "num_tokens": 21420.0,
+      "step": 21
+    },
+    {
+      "entropy": 2.1873691976070404,
+      "epoch": 0.6446886446886447,
+      "grad_norm": 0.8938171863555908,
+      "learning_rate": 0.00016,
+      "loss": 2.1455302238464355,
+      "mean_token_accuracy": 0.5137167796492577,
+      "num_tokens": 22497.0,
+      "step": 22
+    },
+    {
+      "entropy": 2.1501827239990234,
+      "epoch": 0.673992673992674,
+      "grad_norm": 0.9702470898628235,
+      "learning_rate": 0.0001580952380952381,
+      "loss": 2.364546775817871,
+      "mean_token_accuracy": 0.4991302490234375,
+      "num_tokens": 23649.0,
+      "step": 23
+    },
+    {
+      "entropy": 2.1364653259515762,
+      "epoch": 0.7032967032967034,
+      "grad_norm": 0.9650313258171082,
+      "learning_rate": 0.0001561904761904762,
+      "loss": 2.18817400932312,
+      "mean_token_accuracy": 0.5010919496417046,
+      "num_tokens": 24801.0,
+      "step": 24
+    },
+    {
+      "entropy": 2.0752783119678497,
+      "epoch": 0.7326007326007326,
+      "grad_norm": 0.9651536345481873,
+      "learning_rate": 0.0001542857142857143,
+      "loss": 2.2106223106384277,
+      "mean_token_accuracy": 0.5068995915353298,
+      "num_tokens": 25689.0,
+      "step": 25
+    },
+    {
+      "entropy": 2.172010287642479,
+      "epoch": 0.7619047619047619,
+      "grad_norm": 0.897559642791748,
+      "learning_rate": 0.00015238095238095237,
+      "loss": 2.275151014328003,
+      "mean_token_accuracy": 0.5263335853815079,
+      "num_tokens": 26667.0,
+      "step": 26
+    },
+    {
+      "entropy": 2.2060853838920593,
+      "epoch": 0.7912087912087912,
+      "grad_norm": 0.9621699452400208,
+      "learning_rate": 0.00015047619047619048,
+      "loss": 2.229595899581909,
+      "mean_token_accuracy": 0.5082523375749588,
+      "num_tokens": 27656.0,
+      "step": 27
+    },
+    {
+      "entropy": 2.24511581659317,
+      "epoch": 0.8205128205128205,
+      "grad_norm": 0.8171648979187012,
+      "learning_rate": 0.00014857142857142857,
+      "loss": 2.2346324920654297,
+      "mean_token_accuracy": 0.5153078399598598,
+      "num_tokens": 28821.0,
+      "step": 28
+    },
+    {
+      "entropy": 2.2484240978956223,
+      "epoch": 0.8498168498168498,
+      "grad_norm": 0.8533217906951904,
+      "learning_rate": 0.00014666666666666666,
+      "loss": 2.2458832263946533,
+      "mean_token_accuracy": 0.5125137269496918,
+      "num_tokens": 30047.0,
+      "step": 29
+    },
+    {
+      "entropy": 2.2615038454532623,
+      "epoch": 0.8791208791208791,
+      "grad_norm": 0.9642089009284973,
+      "learning_rate": 0.00014476190476190475,
+      "loss": 2.219306468963623,
+      "mean_token_accuracy": 0.520592100918293,
+      "num_tokens": 31006.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.2009344547986984,
+      "epoch": 0.9084249084249084,
+      "grad_norm": 0.9399288296699524,
+      "learning_rate": 0.00014285714285714287,
+      "loss": 2.069526433944702,
+      "mean_token_accuracy": 0.5263588838279247,
+      "num_tokens": 32025.0,
+      "step": 31
+    },
+    {
+      "entropy": 2.1680870354175568,
+      "epoch": 0.9377289377289377,
+      "grad_norm": 0.8993312120437622,
+      "learning_rate": 0.00014095238095238096,
+      "loss": 2.200000524520874,
+      "mean_token_accuracy": 0.5152596533298492,
+      "num_tokens": 32906.0,
+      "step": 32
+    },
+    {
+      "entropy": 2.10675248503685,
+      "epoch": 0.967032967032967,
+      "grad_norm": 0.8099379539489746,
+      "learning_rate": 0.00013904761904761905,
+      "loss": 2.013751268386841,
+      "mean_token_accuracy": 0.5571991391479969,
+      "num_tokens": 33932.0,
+      "step": 33
+    },
+    {
+      "entropy": 2.1444759368896484,
+      "epoch": 0.9963369963369964,
+      "grad_norm": 0.8524768352508545,
+      "learning_rate": 0.00013714285714285716,
+      "loss": 2.191409111022949,
+      "mean_token_accuracy": 0.5195218212902546,
+      "num_tokens": 34923.0,
+      "step": 34
+    },
+    {
+      "entropy": 2.206982374191284,
+      "epoch": 1.0,
+      "grad_norm": 2.50785493850708,
+      "learning_rate": 0.00013523809523809525,
+      "loss": 2.276951551437378,
+      "mean_token_accuracy": 0.5,
+      "num_tokens": 35019.0,
+      "step": 35
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 105,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 260534374821888.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-35/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ba7412e8bcc04ce1aaecd5a483524507e2aac83863d87ce0c7eb227cd9b2bb
+size 5585

checkpoint-70/README.md

@@ -0,0 +1,209 @@
+---
+base_model: liquidai/LFM2-1.2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:liquidai/LFM2-1.2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-70/adapter_config.json

@@ -0,0 +1,47 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "liquidai/LFM2-1.2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "v_proj",
+    "w3",
+    "in_proj",
+    "out_proj",
+    "q_proj",
+    "w2",
+    "w1"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-70/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d0356ece6e4fc822917666f161d2303a82943c70da69645be19e95e90e4fa64
+size 44457856

checkpoint-70/chat_template.jinja

@@ -0,0 +1,37 @@
+{{- bos_token -}}
+{%- set system_prompt = "" -%}
+{%- set ns = namespace(system_prompt="") -%}
+{%- if messages[0]["role"] == "system" -%}
+	{%- set ns.system_prompt = messages[0]["content"] -%}
+	{%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+	{%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%}
+	{%- for tool in tools -%}
+		{%- if tool is not string -%}
+            {%- set tool = tool | tojson -%}
+		{%- endif -%}
+		{%- set ns.system_prompt = ns.system_prompt + tool -%}
+        {%- if not loop.last -%}
+            {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+        {%- endif -%}
+	{%- endfor -%}
+	{%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+	{{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+	{{- "<|im_start|>" + message["role"] + "\n" -}}
+	{%- set content = message["content"] -%}
+	{%- if content is not string -%}
+		{%- set content = content | tojson -%}
+	{%- endif -%}
+	{%- if message["role"] == "tool" -%}
+		{%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%}
+	{%- endif -%}
+	{{- content + "<|im_end|>\n" -}}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+	{{- "<|im_start|>assistant\n" -}}
+{%- endif -%}

checkpoint-70/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c27fe049e8895c3aebfb139ab3c6d4a093e8c4e26239440ccf70a0e879fa674c
+size 22781963

checkpoint-70/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d806e252584cc482a46e06f298ff9c545f593a388b3ea84bc803183719f2e6a7
+size 14645

checkpoint-70/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f57cb2cb9a15d17691efd9341390dee0126b4f7088532f3f4286ff0e2acb9cee
+size 1465

checkpoint-70/tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "legacy": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

checkpoint-70/trainer_state.json

@@ -0,0 +1,734 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 70,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.5899229049682617,
+      "epoch": 0.029304029304029304,
+      "grad_norm": 2.8183693885803223,
+      "learning_rate": 0.0002,
+      "loss": 4.0399041175842285,
+      "mean_token_accuracy": 0.2504686303436756,
+      "num_tokens": 943.0,
+      "step": 1
+    },
+    {
+      "entropy": 2.4779607951641083,
+      "epoch": 0.05860805860805861,
+      "grad_norm": 2.0576627254486084,
+      "learning_rate": 0.0001980952380952381,
+      "loss": 3.5690760612487793,
+      "mean_token_accuracy": 0.33353323861956596,
+      "num_tokens": 1943.0,
+      "step": 2
+    },
+    {
+      "entropy": 2.6818120777606964,
+      "epoch": 0.08791208791208792,
+      "grad_norm": 1.8687777519226074,
+      "learning_rate": 0.0001961904761904762,
+      "loss": 3.2856528759002686,
+      "mean_token_accuracy": 0.3809729106724262,
+      "num_tokens": 2888.0,
+      "step": 3
+    },
+    {
+      "entropy": 2.715443432331085,
+      "epoch": 0.11721611721611722,
+      "grad_norm": 1.5808842182159424,
+      "learning_rate": 0.0001942857142857143,
+      "loss": 2.8003592491149902,
+      "mean_token_accuracy": 0.40646324679255486,
+      "num_tokens": 3931.0,
+      "step": 4
+    },
+    {
+      "entropy": 2.8187700510025024,
+      "epoch": 0.14652014652014653,
+      "grad_norm": 1.4385764598846436,
+      "learning_rate": 0.0001923809523809524,
+      "loss": 2.7211196422576904,
+      "mean_token_accuracy": 0.4199763052165508,
+      "num_tokens": 4819.0,
+      "step": 5
+    },
+    {
+      "entropy": 2.79895681142807,
+      "epoch": 0.17582417582417584,
+      "grad_norm": 1.1686122417449951,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 2.6673367023468018,
+      "mean_token_accuracy": 0.4695504158735275,
+      "num_tokens": 5830.0,
+      "step": 6
+    },
+    {
+      "entropy": 2.7103140354156494,
+      "epoch": 0.20512820512820512,
+      "grad_norm": 1.1704784631729126,
+      "learning_rate": 0.00018857142857142857,
+      "loss": 2.409618854522705,
+      "mean_token_accuracy": 0.5037945993244648,
+      "num_tokens": 6723.0,
+      "step": 7
+    },
+    {
+      "entropy": 2.6644202172756195,
+      "epoch": 0.23443223443223443,
+      "grad_norm": 1.143977165222168,
+      "learning_rate": 0.0001866666666666667,
+      "loss": 2.4525094032287598,
+      "mean_token_accuracy": 0.4569362699985504,
+      "num_tokens": 7724.0,
+      "step": 8
+    },
+    {
+      "entropy": 2.5918588638305664,
+      "epoch": 0.26373626373626374,
+      "grad_norm": 1.000954031944275,
+      "learning_rate": 0.00018476190476190478,
+      "loss": 2.396008253097534,
+      "mean_token_accuracy": 0.4624715633690357,
+      "num_tokens": 8706.0,
+      "step": 9
+    },
+    {
+      "entropy": 2.3777745366096497,
+      "epoch": 0.29304029304029305,
+      "grad_norm": 1.1188299655914307,
+      "learning_rate": 0.00018285714285714286,
+      "loss": 2.281681776046753,
+      "mean_token_accuracy": 0.5069036744534969,
+      "num_tokens": 9652.0,
+      "step": 10
+    },
+    {
+      "entropy": 2.403194159269333,
+      "epoch": 0.32234432234432236,
+      "grad_norm": 0.9767391085624695,
+      "learning_rate": 0.00018095238095238095,
+      "loss": 2.354844331741333,
+      "mean_token_accuracy": 0.4832206554710865,
+      "num_tokens": 10647.0,
+      "step": 11
+    },
+    {
+      "entropy": 2.3256518244743347,
+      "epoch": 0.3516483516483517,
+      "grad_norm": 0.9643223285675049,
+      "learning_rate": 0.00017904761904761907,
+      "loss": 2.2260360717773438,
+      "mean_token_accuracy": 0.5200235471129417,
+      "num_tokens": 11706.0,
+      "step": 12
+    },
+    {
+      "entropy": 2.30185005068779,
+      "epoch": 0.38095238095238093,
+      "grad_norm": 1.1241270303726196,
+      "learning_rate": 0.00017714285714285713,
+      "loss": 2.342158317565918,
+      "mean_token_accuracy": 0.49673087149858475,
+      "num_tokens": 12742.0,
+      "step": 13
+    },
+    {
+      "entropy": 2.2283116281032562,
+      "epoch": 0.41025641025641024,
+      "grad_norm": 1.0168424844741821,
+      "learning_rate": 0.00017523809523809525,
+      "loss": 2.3015754222869873,
+      "mean_token_accuracy": 0.5138893984258175,
+      "num_tokens": 13651.0,
+      "step": 14
+    },
+    {
+      "entropy": 2.251826673746109,
+      "epoch": 0.43956043956043955,
+      "grad_norm": 0.9726622700691223,
+      "learning_rate": 0.00017333333333333334,
+      "loss": 2.2128169536590576,
+      "mean_token_accuracy": 0.5023349672555923,
+      "num_tokens": 14713.0,
+      "step": 15
+    },
+    {
+      "entropy": 2.334069699048996,
+      "epoch": 0.46886446886446886,
+      "grad_norm": 0.9483299255371094,
+      "learning_rate": 0.00017142857142857143,
+      "loss": 2.1633079051971436,
+      "mean_token_accuracy": 0.5210121423006058,
+      "num_tokens": 16110.0,
+      "step": 16
+    },
+    {
+      "entropy": 2.236449494957924,
+      "epoch": 0.4981684981684982,
+      "grad_norm": 0.8546555638313293,
+      "learning_rate": 0.00016952380952380954,
+      "loss": 2.108891725540161,
+      "mean_token_accuracy": 0.5399356037378311,
+      "num_tokens": 17232.0,
+      "step": 17
+    },
+    {
+      "entropy": 2.360167473554611,
+      "epoch": 0.5274725274725275,
+      "grad_norm": 0.9520578384399414,
+      "learning_rate": 0.00016761904761904763,
+      "loss": 2.2965035438537598,
+      "mean_token_accuracy": 0.5066869631409645,
+      "num_tokens": 18287.0,
+      "step": 18
+    },
+    {
+      "entropy": 2.2901051938533783,
+      "epoch": 0.5567765567765568,
+      "grad_norm": 0.8338903188705444,
+      "learning_rate": 0.00016571428571428575,
+      "loss": 2.23210072517395,
+      "mean_token_accuracy": 0.5214172080159187,
+      "num_tokens": 19543.0,
+      "step": 19
+    },
+    {
+      "entropy": 2.297758638858795,
+      "epoch": 0.5860805860805861,
+      "grad_norm": 1.0154532194137573,
+      "learning_rate": 0.0001638095238095238,
+      "loss": 2.3786871433258057,
+      "mean_token_accuracy": 0.4790012463927269,
+      "num_tokens": 20517.0,
+      "step": 20
+    },
+    {
+      "entropy": 2.2084394991397858,
+      "epoch": 0.6153846153846154,
+      "grad_norm": 1.0085352659225464,
+      "learning_rate": 0.00016190476190476192,
+      "loss": 2.2479264736175537,
+      "mean_token_accuracy": 0.4972332492470741,
+      "num_tokens": 21420.0,
+      "step": 21
+    },
+    {
+      "entropy": 2.1873691976070404,
+      "epoch": 0.6446886446886447,
+      "grad_norm": 0.8938171863555908,
+      "learning_rate": 0.00016,
+      "loss": 2.1455302238464355,
+      "mean_token_accuracy": 0.5137167796492577,
+      "num_tokens": 22497.0,
+      "step": 22
+    },
+    {
+      "entropy": 2.1501827239990234,
+      "epoch": 0.673992673992674,
+      "grad_norm": 0.9702470898628235,
+      "learning_rate": 0.0001580952380952381,
+      "loss": 2.364546775817871,
+      "mean_token_accuracy": 0.4991302490234375,
+      "num_tokens": 23649.0,
+      "step": 23
+    },
+    {
+      "entropy": 2.1364653259515762,
+      "epoch": 0.7032967032967034,
+      "grad_norm": 0.9650313258171082,
+      "learning_rate": 0.0001561904761904762,
+      "loss": 2.18817400932312,
+      "mean_token_accuracy": 0.5010919496417046,
+      "num_tokens": 24801.0,
+      "step": 24
+    },
+    {
+      "entropy": 2.0752783119678497,
+      "epoch": 0.7326007326007326,
+      "grad_norm": 0.9651536345481873,
+      "learning_rate": 0.0001542857142857143,
+      "loss": 2.2106223106384277,
+      "mean_token_accuracy": 0.5068995915353298,
+      "num_tokens": 25689.0,
+      "step": 25
+    },
+    {
+      "entropy": 2.172010287642479,
+      "epoch": 0.7619047619047619,
+      "grad_norm": 0.897559642791748,
+      "learning_rate": 0.00015238095238095237,
+      "loss": 2.275151014328003,
+      "mean_token_accuracy": 0.5263335853815079,
+      "num_tokens": 26667.0,
+      "step": 26
+    },
+    {
+      "entropy": 2.2060853838920593,
+      "epoch": 0.7912087912087912,
+      "grad_norm": 0.9621699452400208,
+      "learning_rate": 0.00015047619047619048,
+      "loss": 2.229595899581909,
+      "mean_token_accuracy": 0.5082523375749588,
+      "num_tokens": 27656.0,
+      "step": 27
+    },
+    {
+      "entropy": 2.24511581659317,
+      "epoch": 0.8205128205128205,
+      "grad_norm": 0.8171648979187012,
+      "learning_rate": 0.00014857142857142857,
+      "loss": 2.2346324920654297,
+      "mean_token_accuracy": 0.5153078399598598,
+      "num_tokens": 28821.0,
+      "step": 28
+    },
+    {
+      "entropy": 2.2484240978956223,
+      "epoch": 0.8498168498168498,
+      "grad_norm": 0.8533217906951904,
+      "learning_rate": 0.00014666666666666666,
+      "loss": 2.2458832263946533,
+      "mean_token_accuracy": 0.5125137269496918,
+      "num_tokens": 30047.0,
+      "step": 29
+    },
+    {
+      "entropy": 2.2615038454532623,
+      "epoch": 0.8791208791208791,
+      "grad_norm": 0.9642089009284973,
+      "learning_rate": 0.00014476190476190475,
+      "loss": 2.219306468963623,
+      "mean_token_accuracy": 0.520592100918293,
+      "num_tokens": 31006.0,
+      "step": 30
+    },
+    {
+      "entropy": 2.2009344547986984,
+      "epoch": 0.9084249084249084,
+      "grad_norm": 0.9399288296699524,
+      "learning_rate": 0.00014285714285714287,
+      "loss": 2.069526433944702,
+      "mean_token_accuracy": 0.5263588838279247,
+      "num_tokens": 32025.0,
+      "step": 31
+    },
+    {
+      "entropy": 2.1680870354175568,
+      "epoch": 0.9377289377289377,
+      "grad_norm": 0.8993312120437622,
+      "learning_rate": 0.00014095238095238096,
+      "loss": 2.200000524520874,
+      "mean_token_accuracy": 0.5152596533298492,
+      "num_tokens": 32906.0,
+      "step": 32
+    },
+    {
+      "entropy": 2.10675248503685,
+      "epoch": 0.967032967032967,
+      "grad_norm": 0.8099379539489746,
+      "learning_rate": 0.00013904761904761905,
+      "loss": 2.013751268386841,
+      "mean_token_accuracy": 0.5571991391479969,
+      "num_tokens": 33932.0,
+      "step": 33
+    },
+    {
+      "entropy": 2.1444759368896484,
+      "epoch": 0.9963369963369964,
+      "grad_norm": 0.8524768352508545,
+      "learning_rate": 0.00013714285714285716,
+      "loss": 2.191409111022949,
+      "mean_token_accuracy": 0.5195218212902546,
+      "num_tokens": 34923.0,
+      "step": 34
+    },
+    {
+      "entropy": 2.206982374191284,
+      "epoch": 1.0,
+      "grad_norm": 2.50785493850708,
+      "learning_rate": 0.00013523809523809525,
+      "loss": 2.276951551437378,
+      "mean_token_accuracy": 0.5,
+      "num_tokens": 35019.0,
+      "step": 35
+    },
+    {
+      "entropy": 2.1279809772968292,
+      "epoch": 1.0293040293040292,
+      "grad_norm": 0.812450647354126,
+      "learning_rate": 0.00013333333333333334,
+      "loss": 2.06909441947937,
+      "mean_token_accuracy": 0.5160608440637589,
+      "num_tokens": 36132.0,
+      "step": 36
+    },
+    {
+      "entropy": 2.0765769630670547,
+      "epoch": 1.0586080586080586,
+      "grad_norm": 0.8679144382476807,
+      "learning_rate": 0.00013142857142857143,
+      "loss": 2.101022720336914,
+      "mean_token_accuracy": 0.5235091745853424,
+      "num_tokens": 37176.0,
+      "step": 37
+    },
+    {
+      "entropy": 2.063512533903122,
+      "epoch": 1.0879120879120878,
+      "grad_norm": 1.0040966272354126,
+      "learning_rate": 0.00012952380952380954,
+      "loss": 2.1887731552124023,
+      "mean_token_accuracy": 0.5090154893696308,
+      "num_tokens": 38125.0,
+      "step": 38
+    },
+    {
+      "entropy": 1.9415698498487473,
+      "epoch": 1.1172161172161172,
+      "grad_norm": 0.8333518505096436,
+      "learning_rate": 0.0001276190476190476,
+      "loss": 1.8331354856491089,
+      "mean_token_accuracy": 0.5899114012718201,
+      "num_tokens": 39253.0,
+      "step": 39
+    },
+    {
+      "entropy": 2.115827590227127,
+      "epoch": 1.1465201465201464,
+      "grad_norm": 0.8999753594398499,
+      "learning_rate": 0.00012571428571428572,
+      "loss": 2.0922064781188965,
+      "mean_token_accuracy": 0.5041939504444599,
+      "num_tokens": 40456.0,
+      "step": 40
+    },
+    {
+      "entropy": 2.0162982046604156,
+      "epoch": 1.1758241758241759,
+      "grad_norm": 0.889089822769165,
+      "learning_rate": 0.0001238095238095238,
+      "loss": 1.9345817565917969,
+      "mean_token_accuracy": 0.5448144525289536,
+      "num_tokens": 41595.0,
+      "step": 41
+    },
+    {
+      "entropy": 1.9902137368917465,
+      "epoch": 1.205128205128205,
+      "grad_norm": 0.9763742685317993,
+      "learning_rate": 0.00012190476190476193,
+      "loss": 1.966817855834961,
+      "mean_token_accuracy": 0.5384696908295155,
+      "num_tokens": 42494.0,
+      "step": 42
+    },
+    {
+      "entropy": 1.9661571383476257,
+      "epoch": 1.2344322344322345,
+      "grad_norm": 0.9287217259407043,
+      "learning_rate": 0.00012,
+      "loss": 1.9149142503738403,
+      "mean_token_accuracy": 0.5685045793652534,
+      "num_tokens": 43418.0,
+      "step": 43
+    },
+    {
+      "entropy": 1.8985900431871414,
+      "epoch": 1.2637362637362637,
+      "grad_norm": 0.9483775496482849,
+      "learning_rate": 0.0001180952380952381,
+      "loss": 1.9076926708221436,
+      "mean_token_accuracy": 0.5475717112421989,
+      "num_tokens": 44423.0,
+      "step": 44
+    },
+    {
+      "entropy": 1.9549081772565842,
+      "epoch": 1.293040293040293,
+      "grad_norm": 0.9189373254776001,
+      "learning_rate": 0.00011619047619047621,
+      "loss": 1.9420613050460815,
+      "mean_token_accuracy": 0.5473757795989513,
+      "num_tokens": 45500.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.891863077878952,
+      "epoch": 1.3223443223443223,
+      "grad_norm": 0.9721280932426453,
+      "learning_rate": 0.00011428571428571428,
+      "loss": 1.9221911430358887,
+      "mean_token_accuracy": 0.5582568794488907,
+      "num_tokens": 46546.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.9727875739336014,
+      "epoch": 1.3516483516483517,
+      "grad_norm": 1.0340529680252075,
+      "learning_rate": 0.00011238095238095239,
+      "loss": 1.953711748123169,
+      "mean_token_accuracy": 0.5584920011460781,
+      "num_tokens": 47507.0,
+      "step": 47
+    },
+    {
+      "entropy": 1.8502823263406754,
+      "epoch": 1.380952380952381,
+      "grad_norm": 0.9082620143890381,
+      "learning_rate": 0.00011047619047619049,
+      "loss": 1.883126139640808,
+      "mean_token_accuracy": 0.5711723491549492,
+      "num_tokens": 48476.0,
+      "step": 48
+    },
+    {
+      "entropy": 2.0381594747304916,
+      "epoch": 1.4102564102564101,
+      "grad_norm": 1.0055015087127686,
+      "learning_rate": 0.00010857142857142856,
+      "loss": 2.026916027069092,
+      "mean_token_accuracy": 0.548699252307415,
+      "num_tokens": 49440.0,
+      "step": 49
+    },
+    {
+      "entropy": 1.9274516850709915,
+      "epoch": 1.4395604395604396,
+      "grad_norm": 0.9569126963615417,
+      "learning_rate": 0.00010666666666666667,
+      "loss": 1.9325764179229736,
+      "mean_token_accuracy": 0.5698802806437016,
+      "num_tokens": 50392.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.097727954387665,
+      "epoch": 1.468864468864469,
+      "grad_norm": 0.9394485354423523,
+      "learning_rate": 0.00010476190476190477,
+      "loss": 2.034787654876709,
+      "mean_token_accuracy": 0.5258443988859653,
+      "num_tokens": 51494.0,
+      "step": 51
+    },
+    {
+      "entropy": 1.9933644086122513,
+      "epoch": 1.4981684981684982,
+      "grad_norm": 1.0003395080566406,
+      "learning_rate": 0.00010285714285714286,
+      "loss": 1.834027886390686,
+      "mean_token_accuracy": 0.5819921940565109,
+      "num_tokens": 52453.0,
+      "step": 52
+    },
+    {
+      "entropy": 2.053115800023079,
+      "epoch": 1.5274725274725274,
+      "grad_norm": 1.0380909442901611,
+      "learning_rate": 0.00010095238095238096,
+      "loss": 2.0844743251800537,
+      "mean_token_accuracy": 0.5501821637153625,
+      "num_tokens": 53432.0,
+      "step": 53
+    },
+    {
+      "entropy": 1.8703102469444275,
+      "epoch": 1.5567765567765568,
+      "grad_norm": 1.0789860486984253,
+      "learning_rate": 9.904761904761905e-05,
+      "loss": 1.8772366046905518,
+      "mean_token_accuracy": 0.550880528986454,
+      "num_tokens": 54359.0,
+      "step": 54
+    },
+    {
+      "entropy": 2.006308153271675,
+      "epoch": 1.5860805860805862,
+      "grad_norm": 1.039587140083313,
+      "learning_rate": 9.714285714285715e-05,
+      "loss": 1.9366910457611084,
+      "mean_token_accuracy": 0.5642399489879608,
+      "num_tokens": 55283.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.9818164855241776,
+      "epoch": 1.6153846153846154,
+      "grad_norm": 1.0115448236465454,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 1.959208607673645,
+      "mean_token_accuracy": 0.5404646620154381,
+      "num_tokens": 56360.0,
+      "step": 56
+    },
+    {
+      "entropy": 1.9919450134038925,
+      "epoch": 1.6446886446886446,
+      "grad_norm": 1.0340183973312378,
+      "learning_rate": 9.333333333333334e-05,
+      "loss": 1.8634270429611206,
+      "mean_token_accuracy": 0.563919834792614,
+      "num_tokens": 57270.0,
+      "step": 57
+    },
+    {
+      "entropy": 2.045059621334076,
+      "epoch": 1.673992673992674,
+      "grad_norm": 1.0698320865631104,
+      "learning_rate": 9.142857142857143e-05,
+      "loss": 1.9465467929840088,
+      "mean_token_accuracy": 0.549336139112711,
+      "num_tokens": 58183.0,
+      "step": 58
+    },
+    {
+      "entropy": 2.011963799595833,
+      "epoch": 1.7032967032967035,
+      "grad_norm": 1.0501713752746582,
+      "learning_rate": 8.952380952380953e-05,
+      "loss": 2.0452449321746826,
+      "mean_token_accuracy": 0.5631852373480797,
+      "num_tokens": 59179.0,
+      "step": 59
+    },
+    {
+      "entropy": 1.9702570289373398,
+      "epoch": 1.7326007326007327,
+      "grad_norm": 1.0021151304244995,
+      "learning_rate": 8.761904761904762e-05,
+      "loss": 1.8943660259246826,
+      "mean_token_accuracy": 0.545048289000988,
+      "num_tokens": 60384.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.8890659809112549,
+      "epoch": 1.7619047619047619,
+      "grad_norm": 1.0946227312088013,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 1.9704440832138062,
+      "mean_token_accuracy": 0.569721557199955,
+      "num_tokens": 61392.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.9658374786376953,
+      "epoch": 1.791208791208791,
+      "grad_norm": 1.072704553604126,
+      "learning_rate": 8.380952380952382e-05,
+      "loss": 1.9025211334228516,
+      "mean_token_accuracy": 0.5500420220196247,
+      "num_tokens": 62541.0,
+      "step": 62
+    },
+    {
+      "entropy": 1.9273017197847366,
+      "epoch": 1.8205128205128205,
+      "grad_norm": 1.071745753288269,
+      "learning_rate": 8.19047619047619e-05,
+      "loss": 1.871708869934082,
+      "mean_token_accuracy": 0.5801190733909607,
+      "num_tokens": 63607.0,
+      "step": 63
+    },
+    {
+      "entropy": 2.1368707567453384,
+      "epoch": 1.84981684981685,
+      "grad_norm": 1.0277183055877686,
+      "learning_rate": 8e-05,
+      "loss": 2.0618956089019775,
+      "mean_token_accuracy": 0.5408336892724037,
+      "num_tokens": 64724.0,
+      "step": 64
+    },
+    {
+      "entropy": 1.951027974486351,
+      "epoch": 1.879120879120879,
+      "grad_norm": 1.0923572778701782,
+      "learning_rate": 7.80952380952381e-05,
+      "loss": 1.9516839981079102,
+      "mean_token_accuracy": 0.5601083487272263,
+      "num_tokens": 65691.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.937852904200554,
+      "epoch": 1.9084249084249083,
+      "grad_norm": 1.1021313667297363,
+      "learning_rate": 7.619047619047618e-05,
+      "loss": 1.9671883583068848,
+      "mean_token_accuracy": 0.5423109643161297,
+      "num_tokens": 66741.0,
+      "step": 66
+    },
+    {
+      "entropy": 1.9332957565784454,
+      "epoch": 1.9377289377289377,
+      "grad_norm": 0.963511049747467,
+      "learning_rate": 7.428571428571429e-05,
+      "loss": 1.9559173583984375,
+      "mean_token_accuracy": 0.5629658326506615,
+      "num_tokens": 67942.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.9278145879507065,
+      "epoch": 1.9670329670329672,
+      "grad_norm": 1.0941733121871948,
+      "learning_rate": 7.238095238095238e-05,
+      "loss": 1.785203456878662,
+      "mean_token_accuracy": 0.5560223236680031,
+      "num_tokens": 68867.0,
+      "step": 68
+    },
+    {
+      "entropy": 1.9755759537220001,
+      "epoch": 1.9963369963369964,
+      "grad_norm": 1.2056344747543335,
+      "learning_rate": 7.047619047619048e-05,
+      "loss": 2.001420497894287,
+      "mean_token_accuracy": 0.5633600018918514,
+      "num_tokens": 69898.0,
+      "step": 69
+    },
+    {
+      "entropy": 1.8418163061141968,
+      "epoch": 2.0,
+      "grad_norm": 2.6260459423065186,
+      "learning_rate": 6.857142857142858e-05,
+      "loss": 2.0277798175811768,
+      "mean_token_accuracy": 0.5652173757553101,
+      "num_tokens": 70038.0,
+      "step": 70
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 105,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 519384801822720.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-70/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ba7412e8bcc04ce1aaecd5a483524507e2aac83863d87ce0c7eb227cd9b2bb
+size 5585

tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "legacy": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ba7412e8bcc04ce1aaecd5a483524507e2aac83863d87ce0c7eb227cd9b2bb
+size 5585