bugfix in loading and data processing

config.json

@@ -34,7 +34,7 @@
   "rope_theta": 500000.0,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.47.1",
+  "transformers_version": "4.45.2",
   "use_cache": true,
   "vocab_size": 128256
 }

generation_config.json

@@ -8,5 +8,5 @@
   ],
   "temperature": 0.6,
   "top_p": 0.9,
-  "transformers_version": "4.47.1"
+  "transformers_version": "4.45.2"
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab6434d68ed9cc2d87d40a49848fa8c16b034a495b1fc85fdb58e2fdeb7f853f
+oid sha256:9c1adb90bd346a202a4f08bdabc78a07e0fa3387b00d6f4da5d9db0dc33e74f2
 size 4943274328

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f643977efaea5af706c1973280f6d01c97b346a3a6118f739c3546d1fbcf62d0
+oid sha256:a0724c4783767d7bde60bfc141142ac5a6b68a7ce9b2aca6f3d91e1e5fa06e3e
 size 1050673280

model.safetensors.index.json

@@ -3,7 +3,6 @@
     "total_size": 5993930752
   },
   "weight_map": {
-    "lm_head.weight": "model-00002-of-00002.safetensors",
     "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",

tokenizer_config.json

@@ -2053,7 +2053,6 @@
   "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
-  "extra_special_tokens": {},
   "model_input_names": [
     "input_ids",
     "attention_mask"

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.903225806451613,
+  "epoch": 2.967741935483871,
   "eval_steps": 500,
   "global_step": 69,
   "is_hyper_param_search": false,
@@ -10,103 +10,103 @@
   "log_history": [
     {
       "epoch": 0.21505376344086022,
-      "grad_norm": 4.990048408508301,
+      "grad_norm": 5.693103313446045,
       "learning_rate": 1.9954719225730847e-05,
-      "loss": 1.1226,
+      "loss": 1.1669,
       "step": 5
     },
     {
       "epoch": 0.43010752688172044,
-      "grad_norm": 4.11909294128418,
+      "grad_norm": 1.963355302810669,
       "learning_rate": 1.9450008187146685e-05,
-      "loss": 0.6465,
+      "loss": 0.6522,
       "step": 10
     },
     {
       "epoch": 0.6451612903225806,
-      "grad_norm": 2.001328706741333,
+      "grad_norm": 1.6932282447814941,
       "learning_rate": 1.8412535328311813e-05,
-      "loss": 0.5056,
+      "loss": 0.4912,
       "step": 15
     },
     {
       "epoch": 0.8602150537634409,
-      "grad_norm": 1.5412222146987915,
+      "grad_norm": 1.487468957901001,
       "learning_rate": 1.6900790114821122e-05,
-      "loss": 0.4228,
+      "loss": 0.415,
       "step": 20
     },
     {
-      "epoch": 1.043010752688172,
-      "grad_norm": 1.5938291549682617,
+      "epoch": 1.075268817204301,
+      "grad_norm": 1.5301791429519653,
       "learning_rate": 1.5000000000000002e-05,
-      "loss": 0.3764,
+      "loss": 0.3687,
       "step": 25
     },
     {
-      "epoch": 1.2580645161290323,
-      "grad_norm": 1.3997321128845215,
+      "epoch": 1.2903225806451613,
+      "grad_norm": 1.3023558855056763,
       "learning_rate": 1.2817325568414299e-05,
-      "loss": 0.3061,
+      "loss": 0.2981,
       "step": 30
     },
     {
-      "epoch": 1.4731182795698925,
-      "grad_norm": 1.2927430868148804,
+      "epoch": 1.5053763440860215,
+      "grad_norm": 1.2679604291915894,
       "learning_rate": 1.0475819158237426e-05,
-      "loss": 0.2927,
+      "loss": 0.2836,
       "step": 35
     },
     {
-      "epoch": 1.6881720430107527,
-      "grad_norm": 1.1591598987579346,
+      "epoch": 1.7204301075268817,
+      "grad_norm": 1.180540919303894,
       "learning_rate": 8.107487556395902e-06,
-      "loss": 0.2849,
+      "loss": 0.2772,
       "step": 40
     },
     {
-      "epoch": 1.903225806451613,
-      "grad_norm": 1.0788626670837402,
+      "epoch": 1.935483870967742,
+      "grad_norm": 1.1995508670806885,
       "learning_rate": 5.845849869981137e-06,
-      "loss": 0.2609,
+      "loss": 0.268,
       "step": 45
     },
     {
-      "epoch": 2.086021505376344,
-      "grad_norm": 1.1503716707229614,
+      "epoch": 2.150537634408602,
+      "grad_norm": 1.1225168704986572,
       "learning_rate": 3.818410137793947e-06,
-      "loss": 0.2433,
+      "loss": 0.2306,
       "step": 50
     },
     {
-      "epoch": 2.3010752688172045,
-      "grad_norm": 1.0680569410324097,
+      "epoch": 2.3655913978494625,
+      "grad_norm": 1.166407823562622,
       "learning_rate": 2.1394690525721275e-06,
-      "loss": 0.2139,
+      "loss": 0.2085,
       "step": 55
     },
     {
-      "epoch": 2.5161290322580645,
-      "grad_norm": 1.1041356325149536,
+      "epoch": 2.5806451612903225,
+      "grad_norm": 1.157459020614624,
       "learning_rate": 9.036800464548157e-07,
-      "loss": 0.2041,
+      "loss": 0.2009,
       "step": 60
     },
     {
-      "epoch": 2.731182795698925,
-      "grad_norm": 0.9912173748016357,
+      "epoch": 2.795698924731183,
+      "grad_norm": 1.1147010326385498,
       "learning_rate": 1.8071302737293294e-07,
       "loss": 0.2012,
       "step": 65
     },
     {
-      "epoch": 2.903225806451613,
+      "epoch": 2.967741935483871,
       "step": 69,
-      "total_flos": 2.582932209061069e+16,
-      "train_loss": 0.38006642072097113,
-      "train_runtime": 699.6666,
-      "train_samples_per_second": 6.359,
-      "train_steps_per_second": 0.099
+      "total_flos": 2.640330688875725e+16,
+      "train_loss": 0.37769337063250336,
+      "train_runtime": 647.3014,
+      "train_samples_per_second": 6.873,
+      "train_steps_per_second": 0.107
     }
   ],
   "logging_steps": 5,
@@ -126,7 +126,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.582932209061069e+16,
+  "total_flos": 2.640330688875725e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f06a6c8c59f263f31aa6367582d6ec749e1bcebd5574d2476d628b9bef205ccd
-size 5624
+oid sha256:6c3cc74d9a5d1dacb30e0918d7fae1d54a3a9e4574b16b13b69ba25cc9b3c32b
+size 5496