Hugging Face's logo

taufeeque
/
TinyStories-1Layer-21M-Codebook

Transformers
PyTorch
codebook
Generated from Trainer
Eval Results
Inference Endpoints
Model card Files Community
1
taufeeque commited on
Commit
26ccd30
•
1 Parent(s): b8fba65

upload model

Browse files
Files changed (13) hide show
  1. README.md +175 -0
  2. all_results.json +33 -0
  3. config.json +35 -0
  4. eval_results.json +15 -0
  5. merges.txt +0 -0
  6. pytorch_model.bin +3 -0
  7. special_tokens_map.json +23 -0
  8. tokenizer.json +0 -0
  9. tokenizer_config.json +33 -0
  10. train_results.json +21 -0
  11. trainer_state.json +0 -0
  12. training_args.bin +3 -0
  13. vocab.json +0 -0
README.md ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - generated_from_trainer
4
+ datasets:
5
+ - roneneldan/TinyStories
6
+ metrics:
7
+ - accuracy
8
+ model-index:
9
+ - name: output_main
10
+ results:
11
+ - task:
12
+ name: Causal Language Modeling
13
+ type: text-generation
14
+ dataset:
15
+ name: roneneldan/TinyStories
16
+ type: roneneldan/TinyStories
17
+ metrics:
18
+ - name: Accuracy
19
+ type: accuracy
20
+ value: 0.5791389432485323
21
+ ---
22
+
23
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
24
+ should probably proofread and complete it, then remove this comment. -->
25
+
26
+ # output_main
27
+
28
+ This model is a fine-tuned version of [roneneldan/TinyStories-1Layer-21M](https://huggingface.co/roneneldan/TinyStories-1Layer-21M) on the roneneldan/TinyStories dataset.
29
+ It achieves the following results on the evaluation set:
30
+ - Loss: 1.6604
31
+ - Accuracy: 0.5791
32
+ - Multicode K: 1
33
+ - Dead Code Fraction/layer0: 0.1982
34
+ - Mse/layer0: 6073.8637
35
+ - Input Norm/layer0: 0.7182
36
+ - Output Norm/layer0: 76.7891
37
+
38
+ ## Model description
39
+
40
+ More information needed
41
+
42
+ ## Intended uses & limitations
43
+
44
+ More information needed
45
+
46
+ ## Training and evaluation data
47
+
48
+ More information needed
49
+
50
+ ## Training procedure
51
+
52
+ ### Training hyperparameters
53
+
54
+ The following hyperparameters were used during training:
55
+ - learning_rate: 0.0005
56
+ - train_batch_size: 96
57
+ - eval_batch_size: 64
58
+ - seed: 42
59
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
60
+ - lr_scheduler_type: linear
61
+ - lr_scheduler_warmup_ratio: 0.05
62
+ - training_steps: 100000
63
+
64
+ ### Training results
65
+
66
+ | Training Loss | Epoch | Step | Validation Loss | Accuracy | Multicode K | Dead Code Fraction/layer0 | Mse/layer0 | Input Norm/layer0 | Output Norm/layer0 |
67
+ |:-------------:|:-----:|:------:|:---------------:|:--------:|:-----------:|:-------------------------:|:----------:|:-----------------:|:------------------:|
68
+ | 2.2319 | 0.1 | 1000 | 1.9134 | 0.5317 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
69
+ | 1.8521 | 0.21 | 2000 | 1.7990 | 0.5495 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
70
+ | 1.7879 | 0.31 | 3000 | 1.7739 | 0.5557 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
71
+ | 1.7728 | 0.42 | 4000 | 1.7666 | 0.5564 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
72
+ | 1.7686 | 0.52 | 5000 | 1.7609 | 0.5595 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
73
+ | 1.7635 | 0.63 | 6000 | 1.7555 | 0.5598 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
74
+ | 1.7523 | 0.73 | 7000 | 1.7383 | 0.5632 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
75
+ | 1.7471 | 0.83 | 8000 | 1.7368 | 0.5643 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
76
+ | 1.7404 | 0.94 | 9000 | 1.7277 | 0.5659 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
77
+ | 1.728 | 1.04 | 10000 | 1.7290 | 0.5647 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
78
+ | 1.7195 | 1.15 | 11000 | 1.7244 | 0.5667 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
79
+ | 1.7198 | 1.25 | 12000 | 1.7230 | 0.5671 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
80
+ | 1.7171 | 1.36 | 13000 | 1.7177 | 0.5689 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
81
+ | 1.7185 | 1.46 | 14000 | 1.7150 | 0.5688 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
82
+ | 1.7149 | 1.56 | 15000 | 1.7125 | 0.5695 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
83
+ | 1.7105 | 1.67 | 16000 | 1.7097 | 0.5695 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
84
+ | 1.7107 | 1.77 | 17000 | 1.7073 | 0.5689 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
85
+ | 1.7113 | 1.88 | 18000 | 1.7025 | 0.5712 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
86
+ | 1.7078 | 1.98 | 19000 | 1.7048 | 0.5702 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
87
+ | 1.693 | 2.09 | 20000 | 1.7045 | 0.5696 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
88
+ | 1.6935 | 2.19 | 21000 | 1.7068 | 0.5695 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
89
+ | 1.6962 | 2.29 | 22000 | 1.7046 | 0.5687 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
90
+ | 1.6954 | 2.4 | 23000 | 1.7019 | 0.5706 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
91
+ | 1.6933 | 2.5 | 24000 | 1.7002 | 0.5725 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
92
+ | 1.6942 | 2.61 | 25000 | 1.6983 | 0.5717 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
93
+ | 1.6935 | 2.71 | 26000 | 1.6938 | 0.5730 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
94
+ | 1.6928 | 2.82 | 27000 | 1.6978 | 0.5719 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
95
+ | 1.6927 | 2.92 | 28000 | 1.6935 | 0.5715 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
96
+ | 1.6855 | 3.02 | 29000 | 1.6978 | 0.5726 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
97
+ | 1.6773 | 3.13 | 30000 | 1.6951 | 0.5732 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
98
+ | 1.6788 | 3.23 | 31000 | 1.6926 | 0.5728 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
99
+ | 1.6813 | 3.34 | 32000 | 1.6920 | 0.5726 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
100
+ | 1.6782 | 3.44 | 33000 | 1.6926 | 0.5733 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
101
+ | 1.6801 | 3.55 | 34000 | 1.6894 | 0.5719 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
102
+ | 1.6796 | 3.65 | 35000 | 1.6890 | 0.5728 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
103
+ | 1.6768 | 3.75 | 36000 | 1.6882 | 0.5722 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
104
+ | 1.6802 | 3.86 | 37000 | 1.6872 | 0.5732 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
105
+ | 1.6809 | 3.96 | 38000 | 1.6855 | 0.5750 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
106
+ | 1.6701 | 4.07 | 39000 | 1.6886 | 0.5742 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
107
+ | 1.6646 | 4.17 | 40000 | 1.6890 | 0.5734 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
108
+ | 1.669 | 4.28 | 41000 | 1.6859 | 0.5747 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
109
+ | 1.6713 | 4.38 | 42000 | 1.6867 | 0.5740 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
110
+ | 1.6693 | 4.48 | 43000 | 1.6821 | 0.5750 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
111
+ | 1.6693 | 4.59 | 44000 | 1.6822 | 0.5747 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
112
+ | 1.6692 | 4.69 | 45000 | 1.6801 | 0.5745 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
113
+ | 1.6703 | 4.8 | 46000 | 1.6834 | 0.5761 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
114
+ | 1.6677 | 4.9 | 47000 | 1.6819 | 0.5756 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
115
+ | 1.6682 | 5.01 | 48000 | 1.6778 | 0.5752 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
116
+ | 1.6547 | 5.11 | 49000 | 1.6825 | 0.5751 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
117
+ | 1.6566 | 5.21 | 50000 | 1.6825 | 0.5758 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
118
+ | 1.6605 | 5.32 | 51000 | 1.6814 | 0.5746 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
119
+ | 1.6603 | 5.42 | 52000 | 1.6768 | 0.5755 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
120
+ | 1.6595 | 5.53 | 53000 | 1.6757 | 0.5753 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
121
+ | 1.6603 | 5.63 | 54000 | 1.6769 | 0.5738 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
122
+ | 1.662 | 5.74 | 55000 | 1.6758 | 0.5759 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
123
+ | 1.6602 | 5.84 | 56000 | 1.6771 | 0.5757 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
124
+ | 1.6624 | 5.94 | 57000 | 1.6749 | 0.5770 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
125
+ | 1.6527 | 6.05 | 58000 | 1.6791 | 0.5758 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
126
+ | 1.6474 | 6.15 | 59000 | 1.6763 | 0.5773 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
127
+ | 1.6494 | 6.26 | 60000 | 1.6765 | 0.5761 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
128
+ | 1.6539 | 6.36 | 61000 | 1.6741 | 0.5764 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
129
+ | 1.6539 | 6.47 | 62000 | 1.6752 | 0.5768 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
130
+ | 1.6529 | 6.57 | 63000 | 1.6737 | 0.5775 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
131
+ | 1.6533 | 6.67 | 64000 | 1.6725 | 0.5758 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
132
+ | 1.653 | 6.78 | 65000 | 1.6722 | 0.5774 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
133
+ | 1.6522 | 6.88 | 66000 | 1.6726 | 0.5762 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
134
+ | 1.6528 | 6.99 | 67000 | 1.6726 | 0.5768 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
135
+ | 1.6439 | 7.09 | 68000 | 1.6728 | 0.5771 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
136
+ | 1.6403 | 7.19 | 69000 | 1.6703 | 0.5758 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
137
+ | 1.6447 | 7.3 | 70000 | 1.6697 | 0.5772 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
138
+ | 1.6458 | 7.4 | 71000 | 1.6694 | 0.5777 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
139
+ | 1.6447 | 7.51 | 72000 | 1.6716 | 0.5771 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
140
+ | 1.6449 | 7.61 | 73000 | 1.6680 | 0.5779 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
141
+ | 1.6458 | 7.72 | 74000 | 1.6683 | 0.5779 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
142
+ | 1.6447 | 7.82 | 75000 | 1.6681 | 0.5778 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
143
+ | 1.6451 | 7.92 | 76000 | 1.6677 | 0.5781 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
144
+ | 1.6418 | 8.03 | 77000 | 1.6665 | 0.5789 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
145
+ | 1.6361 | 8.13 | 78000 | 1.6684 | 0.5779 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
146
+ | 1.636 | 8.24 | 79000 | 1.6687 | 0.5786 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
147
+ | 1.6357 | 8.34 | 80000 | 1.6670 | 0.5790 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
148
+ | 1.6379 | 8.45 | 81000 | 1.6658 | 0.5788 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
149
+ | 1.6405 | 8.55 | 82000 | 1.6661 | 0.5788 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
150
+ | 1.6378 | 8.65 | 83000 | 1.6650 | 0.5789 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
151
+ | 1.6386 | 8.76 | 84000 | 1.6650 | 0.5784 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
152
+ | 1.638 | 8.86 | 85000 | 1.6644 | 0.5785 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
153
+ | 1.6374 | 8.97 | 86000 | 1.6635 | 0.5777 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
154
+ | 1.6298 | 9.07 | 87000 | 1.6647 | 0.5785 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
155
+ | 1.6302 | 9.18 | 88000 | 1.6649 | 0.5787 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
156
+ | 1.6315 | 9.28 | 89000 | 1.6651 | 0.5782 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
157
+ | 1.631 | 9.38 | 90000 | 1.6636 | 0.5788 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
158
+ | 1.6316 | 9.49 | 91000 | 1.6627 | 0.5782 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
159
+ | 1.6286 | 9.59 | 92000 | 1.6646 | 0.5783 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
160
+ | 1.6304 | 9.7 | 93000 | 1.6632 | 0.5801 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
161
+ | 1.6298 | 9.8 | 94000 | 1.6623 | 0.5800 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
162
+ | 1.6309 | 9.91 | 95000 | 1.6620 | 0.5800 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
163
+ | 1.6302 | 10.01 | 96000 | 1.6602 | 0.5801 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
164
+ | 1.6242 | 10.11 | 97000 | 1.6610 | 0.5786 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
165
+ | 1.6258 | 10.22 | 98000 | 1.6605 | 0.5795 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
166
+ | 1.6234 | 10.32 | 99000 | 1.6605 | 0.5791 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
167
+ | 1.6245 | 10.43 | 100000 | 1.6604 | 0.5791 | 1 | 1.0 | 0.0 | 0.0 | 0.0 |
168
+
169
+
170
+ ### Framework versions
171
+
172
+ - Transformers 4.29.2
173
+ - Pytorch 2.0.1+cu117
174
+ - Datasets 2.12.0
175
+ - Tokenizers 0.13.3
all_results.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "MSE": 0.0,
3
+ "MSE/layer0": 0.0,
4
+ "dead_code_fraction": 1.0,
5
+ "dead_code_fraction/layer0": 1.0,
6
+ "epoch": 10.43,
7
+ "eval_MSE/layer0": 6073.8636798095695,
8
+ "eval_accuracy": 0.5791389432485323,
9
+ "eval_dead_code_fraction/layer0": 0.1981725,
10
+ "eval_input_norm/layer0": 0.7182212994247673,
11
+ "eval_loss": 1.6604058742523193,
12
+ "eval_multicode_k": 1,
13
+ "eval_output_norm/layer0": 76.78913438796998,
14
+ "eval_runtime": 6.7146,
15
+ "eval_samples": 100,
16
+ "eval_samples_per_second": 14.893,
17
+ "eval_steps_per_second": 0.298,
18
+ "input_norm": 0.0,
19
+ "input_norm/layer0": 0.0,
20
+ "loss": 1.6774777018260956,
21
+ "max_norm": 153.29054260253906,
22
+ "max_norm/layer0": 153.29054260253906,
23
+ "mean_norm": 75.17323780059814,
24
+ "mean_norm/layer0": 75.17323780059814,
25
+ "multicode_k": 1,
26
+ "output_norm": 0.0,
27
+ "output_norm/layer0": 0.0,
28
+ "perplexity": 5.261445896555633,
29
+ "runtime": 132212.7109,
30
+ "samples_per_second": 72.61,
31
+ "steps_per_second": 0.756,
32
+ "train_samples": 920563
33
+ }
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "GPTNeoCodebookModel"
4
+ ],
5
+ "codebook_at": [
6
+ "preproj_attention"
7
+ ],
8
+ "codebook_kwargs": {},
9
+ "codebook_type": [
10
+ "compositional"
11
+ ],
12
+ "k_codebook": [
13
+ 8
14
+ ],
15
+ "kmeans_init": false,
16
+ "kmeans_init_examples": 1000,
17
+ "kmeans_kwargs": {
18
+ "batch_size": 24576,
19
+ "n_init": "auto"
20
+ },
21
+ "kmeans_path": "/.cache/cb_volume/huggingface/kmeans_embeddings.pt",
22
+ "layers_to_snap": [
23
+ 0
24
+ ],
25
+ "loss": "aeloss",
26
+ "model_type": "codebook",
27
+ "num_codebooks": [
28
+ 16
29
+ ],
30
+ "num_codes": 25000,
31
+ "replace_codes": false,
32
+ "similarity_metric": "inner_product",
33
+ "torch_dtype": "float32",
34
+ "transformers_version": "4.29.2"
35
+ }
eval_results.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.43,
3
+ "eval_MSE/layer0": 6073.8636798095695,
4
+ "eval_accuracy": 0.5791389432485323,
5
+ "eval_dead_code_fraction/layer0": 0.1981725,
6
+ "eval_input_norm/layer0": 0.7182212994247673,
7
+ "eval_loss": 1.6604058742523193,
8
+ "eval_multicode_k": 1,
9
+ "eval_output_norm/layer0": 76.78913438796998,
10
+ "eval_runtime": 6.7146,
11
+ "eval_samples": 100,
12
+ "eval_samples_per_second": 14.893,
13
+ "eval_steps_per_second": 0.298,
14
+ "perplexity": 5.261445896555633
15
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17bb9795f1e6739c83f2bbabcb103948ef59f25f577abc1ab5ccb094f65bad95
3
+ size 371248378
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<|endoftext|>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": {
14
+ "__type": "AddedToken",
15
+ "content": "<|endoftext|>",
16
+ "lstrip": false,
17
+ "normalized": true,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "errors": "replace",
22
+ "model_max_length": 2048,
23
+ "pad_token": null,
24
+ "tokenizer_class": "GPT2Tokenizer",
25
+ "unk_token": {
26
+ "__type": "AddedToken",
27
+ "content": "<|endoftext|>",
28
+ "lstrip": false,
29
+ "normalized": true,
30
+ "rstrip": false,
31
+ "single_word": false
32
+ }
33
+ }
train_results.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "MSE": 0.0,
3
+ "MSE/layer0": 0.0,
4
+ "dead_code_fraction": 1.0,
5
+ "dead_code_fraction/layer0": 1.0,
6
+ "epoch": 10.43,
7
+ "input_norm": 0.0,
8
+ "input_norm/layer0": 0.0,
9
+ "loss": 1.6774777018260956,
10
+ "max_norm": 153.29054260253906,
11
+ "max_norm/layer0": 153.29054260253906,
12
+ "mean_norm": 75.17323780059814,
13
+ "mean_norm/layer0": 75.17323780059814,
14
+ "multicode_k": 1,
15
+ "output_norm": 0.0,
16
+ "output_norm/layer0": 0.0,
17
+ "runtime": 132212.7109,
18
+ "samples_per_second": 72.61,
19
+ "steps_per_second": 0.756,
20
+ "train_samples": 920563
21
+ }
trainer_state.json ADDED
The diff for this file is too large to render. See raw diff
 
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:694d424d45513c05a84b6e17c1f77f79366a09ef35a8b910241185856f3ede97
3
+ size 4155
vocab.json ADDED
The diff for this file is too large to render. See raw diff