Josephgflowers commited on
Commit
2563639
1 Parent(s): ca1b0cf

Upload 12 files

Browse files
config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/home/joe/Downloads/so_much-math",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 2048,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 5632,
14
+ "max_position_embeddings": 2048,
15
+ "model_type": "llama",
16
+ "num_attention_heads": 32,
17
+ "num_hidden_layers": 31,
18
+ "num_key_value_heads": 4,
19
+ "pretraining_tp": 1,
20
+ "rms_norm_eps": 1e-05,
21
+ "rope_scaling": null,
22
+ "rope_theta": 10000.0,
23
+ "tie_word_embeddings": false,
24
+ "torch_dtype": "float16",
25
+ "transformers_version": "4.38.0.dev0",
26
+ "unsloth_version": "2024.1",
27
+ "use_cache": false,
28
+ "vocab_size": 32000
29
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.38.0.dev0",
6
+ "use_cache": false
7
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:484bb6bf80f750e9ccc7e70ad2aafa4c03576420189023a7aa98102c21b72586
3
+ size 2992926176
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b58273a18857dd12b639dc6c4e27b6df1e608156ab00404b49e01cc9d5c62065
3
+ size 5454522
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
3
+ size 14244
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2f18c8a721b3db6249c3f6a7e3fbe6262f13cef0416eb6f736e7eb3b5ab63271
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "</s>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "bos_token": "<s>",
31
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": false,
35
+ "max_length": 2048,
36
+ "model_max_length": 2048,
37
+ "pad_token": "</s>",
38
+ "padding_side": "right",
39
+ "sp_model_kwargs": {},
40
+ "stride": 0,
41
+ "tokenizer_class": "LlamaTokenizer",
42
+ "truncation_side": "right",
43
+ "truncation_strategy": "longest_first",
44
+ "unk_token": "<unk>",
45
+ "use_default_system_prompt": false
46
+ }
trainer_state.json ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.2705993776214315,
5
+ "eval_steps": 500,
6
+ "global_step": 2000,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.01,
13
+ "grad_norm": 0.8518972396850586,
14
+ "learning_rate": 4.933026654038696e-05,
15
+ "loss": 0.5532,
16
+ "step": 100
17
+ },
18
+ {
19
+ "epoch": 0.03,
20
+ "grad_norm": 0.7409780621528625,
21
+ "learning_rate": 4.865376809633338e-05,
22
+ "loss": 0.5033,
23
+ "step": 200
24
+ },
25
+ {
26
+ "epoch": 0.04,
27
+ "grad_norm": 0.7067281007766724,
28
+ "learning_rate": 4.79772696522798e-05,
29
+ "loss": 0.4602,
30
+ "step": 300
31
+ },
32
+ {
33
+ "epoch": 0.05,
34
+ "grad_norm": 0.7321436405181885,
35
+ "learning_rate": 4.7300771208226225e-05,
36
+ "loss": 0.4523,
37
+ "step": 400
38
+ },
39
+ {
40
+ "epoch": 0.07,
41
+ "grad_norm": 0.6380289793014526,
42
+ "learning_rate": 4.6624272764172644e-05,
43
+ "loss": 0.4606,
44
+ "step": 500
45
+ },
46
+ {
47
+ "epoch": 0.08,
48
+ "grad_norm": 0.5421116352081299,
49
+ "learning_rate": 4.594777432011906e-05,
50
+ "loss": 0.4262,
51
+ "step": 600
52
+ },
53
+ {
54
+ "epoch": 0.09,
55
+ "grad_norm": 0.5193295478820801,
56
+ "learning_rate": 4.527127587606549e-05,
57
+ "loss": 0.4105,
58
+ "step": 700
59
+ },
60
+ {
61
+ "epoch": 0.11,
62
+ "grad_norm": 0.5962494611740112,
63
+ "learning_rate": 4.459477743201191e-05,
64
+ "loss": 0.4174,
65
+ "step": 800
66
+ },
67
+ {
68
+ "epoch": 0.12,
69
+ "grad_norm": 0.6797559261322021,
70
+ "learning_rate": 4.391827898795833e-05,
71
+ "loss": 0.4346,
72
+ "step": 900
73
+ },
74
+ {
75
+ "epoch": 0.14,
76
+ "grad_norm": 0.5834950804710388,
77
+ "learning_rate": 4.324178054390475e-05,
78
+ "loss": 0.4233,
79
+ "step": 1000
80
+ },
81
+ {
82
+ "epoch": 0.15,
83
+ "grad_norm": 0.5665029287338257,
84
+ "learning_rate": 4.256528209985117e-05,
85
+ "loss": 0.4119,
86
+ "step": 1100
87
+ },
88
+ {
89
+ "epoch": 0.16,
90
+ "grad_norm": 0.5659196972846985,
91
+ "learning_rate": 4.188878365579759e-05,
92
+ "loss": 0.4242,
93
+ "step": 1200
94
+ },
95
+ {
96
+ "epoch": 0.18,
97
+ "grad_norm": 0.5055731534957886,
98
+ "learning_rate": 4.121228521174402e-05,
99
+ "loss": 0.4075,
100
+ "step": 1300
101
+ },
102
+ {
103
+ "epoch": 0.19,
104
+ "grad_norm": 0.5024439692497253,
105
+ "learning_rate": 4.053578676769044e-05,
106
+ "loss": 0.3856,
107
+ "step": 1400
108
+ },
109
+ {
110
+ "epoch": 0.2,
111
+ "grad_norm": 0.5468323826789856,
112
+ "learning_rate": 3.9859288323636856e-05,
113
+ "loss": 0.3942,
114
+ "step": 1500
115
+ },
116
+ {
117
+ "epoch": 0.22,
118
+ "grad_norm": 0.579856812953949,
119
+ "learning_rate": 3.918278987958328e-05,
120
+ "loss": 0.3914,
121
+ "step": 1600
122
+ },
123
+ {
124
+ "epoch": 0.23,
125
+ "grad_norm": 0.6119765043258667,
126
+ "learning_rate": 3.85062914355297e-05,
127
+ "loss": 0.4195,
128
+ "step": 1700
129
+ },
130
+ {
131
+ "epoch": 0.24,
132
+ "grad_norm": 0.4361695647239685,
133
+ "learning_rate": 3.782979299147612e-05,
134
+ "loss": 0.3926,
135
+ "step": 1800
136
+ },
137
+ {
138
+ "epoch": 0.26,
139
+ "grad_norm": 0.4878581464290619,
140
+ "learning_rate": 3.715329454742254e-05,
141
+ "loss": 0.3738,
142
+ "step": 1900
143
+ },
144
+ {
145
+ "epoch": 0.27,
146
+ "grad_norm": 0.6861026883125305,
147
+ "learning_rate": 3.6476796103368965e-05,
148
+ "loss": 0.4021,
149
+ "step": 2000
150
+ }
151
+ ],
152
+ "logging_steps": 100,
153
+ "max_steps": 7391,
154
+ "num_input_tokens_seen": 0,
155
+ "num_train_epochs": 1,
156
+ "save_steps": 1000,
157
+ "total_flos": 5.62657090338816e+17,
158
+ "train_batch_size": 16,
159
+ "trial_name": null,
160
+ "trial_params": null
161
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c08f18a63de89b1f80f499629968399a4fbd3284e2375345b0c62a39936aae5
3
+ size 4920