Training in progress, step 100

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "btherien/410M_it-132366_Tr-slim-pajama-300B-replay5_finetune_sft-full",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "rope_scaling": null,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.2",
+  "use_cache": false,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e6ff88115be01c2c7e55983bd5f414cee937fe24b777c87b7ae207b597f6d43
+size 810702192

runs/Mar05_09-39-34_u124757/events.out.tfevents.1709660817.u124757.2428169.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:658b49dadd76d5ab09a9e1279da7e54ed4182f69580b04af13f016d1c77c54af
+size 4788

runs/Mar05_09-47-38_u124757/events.out.tfevents.1709661072.u124757.2436583.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd0cbecf4d290a4e9a36febfbaf30dfc120a98f90b55e6652fcabaf35a86b3a7
+size 4788

runs/Mar05_09-51-53_u124757/events.out.tfevents.1709661325.u124757.2441023.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b32b25674a32a8fb1f69f168fe02fc6f5c6053836a65bcb727033431539a671d
+size 4788

runs/Mar05_09-58-46_u124757/events.out.tfevents.1709661738.u124757.2452648.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc63e499b9a3ba762f0e092874477e52007130442f95902bdef0eee01bf70cc6
+size 4788

runs/Mar05_10-16-01_u124757/events.out.tfevents.1709662774.u124757.2467438.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a905e68297356c06fe7c9873856a5991c6290ad18a03aeae20137d9a081b05
+size 4788

runs/Mar05_11-07-12_u124757/events.out.tfevents.1709665844.u124757.2510347.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7ab15a2d32de7487d120157206898ad79100e9b64f0a3bd4fb2d0a5b322edf8
+size 4788

runs/Mar05_11-15-57_u124757/events.out.tfevents.1709666367.u124757.2521270.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48fdb5ee57f38e666dc0ac7f754f34394923e73758f11a3740c54bfd22ab669f
+size 4788

runs/Mar05_11-24-14_u124757/events.out.tfevents.1709666910.u124757.2640297.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cbd996df3a2535a1a081f3403f0c43b889856d663c71b7562c49eacba350ab1
+size 4788

runs/Mar05_11-47-42_u124757/events.out.tfevents.1709668307.u124757.3058420.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a725daa28eb463a2110cecef5be2dc3ce931e9719fd36fd6eb9fc331938a1b1
+size 4788

runs/Mar05_11-55-55_u124757/events.out.tfevents.1709668922.u124757.3202984.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a79426c651539e49f11a76abf1aa5d2a3708d858511102731231b5babde3be1f
+size 4788

runs/Mar05_12-04-17_u124757/events.out.tfevents.1709669293.u124757.3356547.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e036b22b702c11617e815d600747e5e657e95686651d9db99898bd8b068289a6
+size 4788

runs/Mar05_12-35-26_u124757/events.out.tfevents.1709671163.u124757.3927336.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b04c257b29f4eea1d4cf1446a58db5b97589643454667459817fdb3088032cf2
+size 4788

runs/Mar05_12-49-12_u124757/events.out.tfevents.1709672122.u124757.4177068.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a904f92a75c9dbd7f0ece2285719861065ba7831b6d8037562c6ee6657d3f92
+size 4788

runs/Mar05_13-00-22_u124757/events.out.tfevents.1709672666.u124757.188112.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e316ae762a2681d3b17ab47484fafa327cebfb177317da353484da45a002d92
+size 6657

runs/Mar05_13-08-21_u124757/events.out.tfevents.1709673156.u124757.331914.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:734843b02a7d88cbe9b593573eaf243d9ffcc163cc7f8936eb1748854366519c
+size 12368

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "bos_token": {
+    "content": "<|padding|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<|padding|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": "<|endoftext|>",
+  "mask_token": {
+    "content": "<|padding|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|padding|>",
+  "sep_token": {
+    "content": "<|padding|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|padding|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,215 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|padding|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + '<|endoftext|>' }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + '<|endoftext|>' }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + '<|endoftext|>' }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<|padding|>",
+  "eos_token": "<|endoftext|>",
+  "mask_token": "<|padding|>",
+  "model_max_length": 2048,
+  "pad_token": "<|padding|>",
+  "sep_token": "<|padding|>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<|padding|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cc2acb47689f7b21f23745899d2a935cbfbb6cfb0cbc69eb749f6ab508e9052
+size 5435