Hugging Face's logo

0xsuid
/
GPT-Neo-125M

TensorBoard
Model card Files Metrics Community
0xsuid commited on
Commit
662c2cd
·
1 Parent(s): 0f59e3f

Upload experiments/ with huggingface_hub

Browse files
Files changed (48) hide show
  1. experiments/configs.json +26 -0
  2. experiments/deepspeed.json +48 -0
  3. experiments/final_checkpoint/config.json +54 -0
  4. experiments/final_checkpoint/pytorch_model.bin +3 -0
  5. experiments/final_checkpoint/tokenizer/merges.txt +0 -0
  6. experiments/final_checkpoint/tokenizer/special_tokens_map.json +24 -0
  7. experiments/final_checkpoint/tokenizer/tokenizer.json +0 -0
  8. experiments/final_checkpoint/tokenizer/tokenizer_config.json +34 -0
  9. experiments/final_checkpoint/tokenizer/vocab.json +0 -0
  10. experiments/logs/1676990060.264247/events.out.tfevents.1676990060.7b1eed3780f4.2829.2 +3 -0
  11. experiments/logs/1676990154.028005/events.out.tfevents.1676990154.7b1eed3780f4.5567.2 +3 -0
  12. experiments/logs/1676990712.4404476/events.out.tfevents.1676990712.7b1eed3780f4.10223.2 +3 -0
  13. experiments/logs/1677006993.51447/events.out.tfevents.1677006993.7b1eed3780f4.46131.2 +3 -0
  14. experiments/logs/1677007417.411844/events.out.tfevents.1677007417.7b1eed3780f4.51624.2 +3 -0
  15. experiments/logs/1677007657.7754965/events.out.tfevents.1677007657.7b1eed3780f4.56432.2 +3 -0
  16. experiments/logs/1677007781.5676754/events.out.tfevents.1677007781.7b1eed3780f4.59255.2 +3 -0
  17. experiments/logs/1677007927.0654259/events.out.tfevents.1677007927.7b1eed3780f4.62309.2 +3 -0
  18. experiments/logs/1677008561.9460855/events.out.tfevents.1677008561.7b1eed3780f4.3852.2 +3 -0
  19. experiments/logs/1677009134.735858/events.out.tfevents.1677009134.7b1eed3780f4.11652.2 +3 -0
  20. experiments/logs/events.out.tfevents.1676990060.7b1eed3780f4.2829.1 +3 -0
  21. experiments/logs/events.out.tfevents.1676990154.7b1eed3780f4.5567.1 +3 -0
  22. experiments/logs/events.out.tfevents.1676990712.7b1eed3780f4.10223.1 +3 -0
  23. experiments/logs/events.out.tfevents.1677006993.7b1eed3780f4.46131.1 +3 -0
  24. experiments/logs/events.out.tfevents.1677007417.7b1eed3780f4.51624.1 +3 -0
  25. experiments/logs/events.out.tfevents.1677007657.7b1eed3780f4.56432.1 +3 -0
  26. experiments/logs/events.out.tfevents.1677007781.7b1eed3780f4.59255.1 +3 -0
  27. experiments/logs/events.out.tfevents.1677007927.7b1eed3780f4.62309.1 +3 -0
  28. experiments/logs/events.out.tfevents.1677008561.7b1eed3780f4.3852.1 +3 -0
  29. experiments/logs/events.out.tfevents.1677009134.7b1eed3780f4.11652.1 +3 -0
  30. experiments/logs/train_neo/events.out.tfevents.1676989999.7b1eed3780f4.2829.0 +3 -0
  31. experiments/logs/train_neo/events.out.tfevents.1676990139.7b1eed3780f4.5567.0 +3 -0
  32. experiments/logs/train_neo/events.out.tfevents.1676990697.7b1eed3780f4.10223.0 +3 -0
  33. experiments/logs/train_neo/events.out.tfevents.1677006978.7b1eed3780f4.46131.0 +3 -0
  34. experiments/logs/train_neo/events.out.tfevents.1677007402.7b1eed3780f4.51624.0 +3 -0
  35. experiments/logs/train_neo/events.out.tfevents.1677007642.7b1eed3780f4.56432.0 +3 -0
  36. experiments/logs/train_neo/events.out.tfevents.1677007765.7b1eed3780f4.59255.0 +3 -0
  37. experiments/logs/train_neo/events.out.tfevents.1677007911.7b1eed3780f4.62309.0 +3 -0
  38. experiments/logs/train_neo/events.out.tfevents.1677008546.7b1eed3780f4.3852.0 +3 -0
  39. experiments/logs/train_neo/events.out.tfevents.1677009119.7b1eed3780f4.11652.0 +3 -0
  40. experiments/output.log +0 -0
  41. experiments/trainer_final_checkpoint/config.json +54 -0
  42. experiments/trainer_final_checkpoint/merges.txt +0 -0
  43. experiments/trainer_final_checkpoint/pytorch_model.bin +3 -0
  44. experiments/trainer_final_checkpoint/special_tokens_map.json +24 -0
  45. experiments/trainer_final_checkpoint/tokenizer.json +0 -0
  46. experiments/trainer_final_checkpoint/tokenizer_config.json +34 -0
  47. experiments/trainer_final_checkpoint/training_args.bin +3 -0
  48. experiments/trainer_final_checkpoint/vocab.json +0 -0
experiments/configs.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_dir": "./results",
3
+ "evaluation_strategy": "no",
4
+ "do_eval": false,
5
+ "eval_steps": 0,
6
+ "log_level": "info",
7
+ "logging_first_step": true,
8
+ "logging_steps": 5,
9
+ "logging_dir": "./logs",
10
+ "save_steps": 150,
11
+ "save_total_limit": 1,
12
+ "num_train_epochs": 10,
13
+ "per_device_train_batch_size": 6,
14
+ "optim": "adamw_torch",
15
+ "gradient_accumulation_steps": 4,
16
+ "dataloader_drop_last": true,
17
+ "warmup_steps": 1000,
18
+ "weight_decay": 0.1,
19
+ "learning_rate": 0.0001,
20
+ "deepspeed": "deepspeed.json",
21
+ "local_rank": 0,
22
+ "total_gpus": 8,
23
+ "v_cpus": 64,
24
+ "total_memory_in_gb": 257604.9765625,
25
+ "dataset_limit": 0
26
+ }
experiments/deepspeed.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "optimizer": {
3
+ "type": "AdamW",
4
+ "params": {
5
+ "lr": 1e-4,
6
+ "betas": [ 0.9, 0.999 ],
7
+ "eps": 1e-8,
8
+ "weight_decay": 0.1
9
+ }
10
+ },
11
+
12
+ "scheduler": {
13
+ "type": "WarmupLR",
14
+ "params": {
15
+ "warmup_min_lr": 1e-9,
16
+ "warmup_max_lr": 1e-4,
17
+ "warmup_num_steps": 1000
18
+ }
19
+ },
20
+
21
+ "zero_optimization": {
22
+ "stage": 2,
23
+ "offload_optimizer": {
24
+ "device": "cpu",
25
+ "pin_memory": true
26
+ },
27
+ "allgather_partitions": true,
28
+ "allgather_bucket_size": 5e8,
29
+ "overlap_comm": true,
30
+ "reduce_scatter": true,
31
+ "reduce_bucket_size": 5e8,
32
+ "contiguous_gradients": true
33
+ },
34
+
35
+ "tensorboard": {
36
+ "enabled": true,
37
+ "output_path": "logs/",
38
+ "job_name": "train_neo"
39
+ },
40
+
41
+ "zero_allow_untested_optimizer": true,
42
+ "gradient_accumulation_steps": "auto",
43
+ "gradient_clipping": "auto",
44
+ "steps_per_print": 2000,
45
+ "train_batch_size": "auto",
46
+ "train_micro_batch_size_per_gpu": "auto",
47
+ "wall_clock_breakdown": false
48
+ }
experiments/final_checkpoint/config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "EleutherAI/gpt-neo-125M",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPTNeoForCausalLM"
6
+ ],
7
+ "attention_dropout": 0,
8
+ "attention_layers": [
9
+ "global",
10
+ "local",
11
+ "global",
12
+ "local",
13
+ "global",
14
+ "local",
15
+ "global",
16
+ "local",
17
+ "global",
18
+ "local",
19
+ "global",
20
+ "local"
21
+ ],
22
+ "attention_types": [
23
+ [
24
+ [
25
+ "global",
26
+ "local"
27
+ ],
28
+ 6
29
+ ]
30
+ ],
31
+ "bos_token_id": 50256,
32
+ "embed_dropout": 0,
33
+ "eos_token_id": 50256,
34
+ "gradient_checkpointing": false,
35
+ "hidden_size": 768,
36
+ "initializer_range": 0.02,
37
+ "intermediate_size": null,
38
+ "layer_norm_epsilon": 1e-05,
39
+ "max_position_embeddings": 2048,
40
+ "model_type": "gpt_neo",
41
+ "num_heads": 12,
42
+ "num_layers": 12,
43
+ "resid_dropout": 0,
44
+ "summary_activation": null,
45
+ "summary_first_dropout": 0.1,
46
+ "summary_proj_to_labels": true,
47
+ "summary_type": "cls_index",
48
+ "summary_use_proj": true,
49
+ "torch_dtype": "float32",
50
+ "transformers_version": "4.24.0",
51
+ "use_cache": true,
52
+ "vocab_size": 50257,
53
+ "window_size": 256
54
+ }
experiments/final_checkpoint/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6589651dd8c9588a8faa97daec7814ffcfe998ba2e481eabdcb3e75405761d5d
3
+ size 551154684
experiments/final_checkpoint/tokenizer/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
experiments/final_checkpoint/tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<|endoftext|>",
17
+ "unk_token": {
18
+ "content": "<|endoftext|>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
experiments/final_checkpoint/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
experiments/final_checkpoint/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "eos_token": {
13
+ "__type": "AddedToken",
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ },
20
+ "errors": "replace",
21
+ "model_max_length": 2048,
22
+ "name_or_path": "EleutherAI/gpt-neo-125M",
23
+ "pad_token": null,
24
+ "special_tokens_map_file": null,
25
+ "tokenizer_class": "GPT2Tokenizer",
26
+ "unk_token": {
27
+ "__type": "AddedToken",
28
+ "content": "<|endoftext|>",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false
33
+ }
34
+ }
experiments/final_checkpoint/tokenizer/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
experiments/logs/1676990060.264247/events.out.tfevents.1676990060.7b1eed3780f4.2829.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0a44882a6cf48f00bbae625e93c4eb24bf47ac0723da1b60bba90e403527478
3
+ size 5418
experiments/logs/1676990154.028005/events.out.tfevents.1676990154.7b1eed3780f4.5567.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8cad42700950e422799625ed456e575bab05a5a8d59a135ad217bc04118fa3a
3
+ size 5418
experiments/logs/1676990712.4404476/events.out.tfevents.1676990712.7b1eed3780f4.10223.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:604efe07f7a9c4930e77e489c3df413af1a8323edd27cdee3d2f1d128f4c1c89
3
+ size 5418
experiments/logs/1677006993.51447/events.out.tfevents.1677006993.7b1eed3780f4.46131.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a22c2f7b4fbb4cd79ea6415d3c51f9a09dc73db339caa3f5f5eb586fe1302e8a
3
+ size 5418
experiments/logs/1677007417.411844/events.out.tfevents.1677007417.7b1eed3780f4.51624.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8abe3ac2c851eefaa9ea98dbcabd1ddf10d0de80c47b521651f31e3a50b2b785
3
+ size 5418
experiments/logs/1677007657.7754965/events.out.tfevents.1677007657.7b1eed3780f4.56432.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fd51f599cb26d493f515bc23292f9207bc3c12926fd96a21adef93e0d48ab7a
3
+ size 5418
experiments/logs/1677007781.5676754/events.out.tfevents.1677007781.7b1eed3780f4.59255.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f59371ae067572e2bac707ffc12c5a6b195952c9f8a02efd1f13071407fdef34
3
+ size 5418
experiments/logs/1677007927.0654259/events.out.tfevents.1677007927.7b1eed3780f4.62309.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:583410e80db49e1fd00d700134bc0c7b940a282a02db4ea1c04e343b2cf95d53
3
+ size 5418
experiments/logs/1677008561.9460855/events.out.tfevents.1677008561.7b1eed3780f4.3852.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f9718ff8c295f15850eae9e52a47d89e4fe679205f0878dbbebae43b69e924e
3
+ size 5418
experiments/logs/1677009134.735858/events.out.tfevents.1677009134.7b1eed3780f4.11652.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de0699c3ccb09a5e8169901a5a82bf8626cdb9173b5ef41fa9b1ed6130d9e876
3
+ size 5418
experiments/logs/events.out.tfevents.1676990060.7b1eed3780f4.2829.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d60506ce0259929b66d97f59ae51e9741a0439206a3e90b9eafafbb1d4e55d99
3
+ size 4972
experiments/logs/events.out.tfevents.1676990154.7b1eed3780f4.5567.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c40f4b630877ec383adc308e311d8a0055d151bdba93a915ac8627ac21f703b
3
+ size 7128
experiments/logs/events.out.tfevents.1676990712.7b1eed3780f4.10223.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc84a6719de976bab0c9c89d033c452a0d36fe758d4ea7b29b7744044ba17bd8
3
+ size 93460
experiments/logs/events.out.tfevents.1677006993.7b1eed3780f4.46131.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2f8491b10d8281d0ca19c6f340ada66b121750b085ed36267b6d7915a0a7d100
3
+ size 5126
experiments/logs/events.out.tfevents.1677007417.7b1eed3780f4.51624.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9138322d0249bf3e3664827b969c7e1ba7ecfeed12561218b0107aff49d5db6
3
+ size 4663
experiments/logs/events.out.tfevents.1677007657.7b1eed3780f4.56432.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82770df39e009cdd002b40f4ef5a003d0c10180e1d9e9cff927bdfcc28829f0f
3
+ size 4356
experiments/logs/events.out.tfevents.1677007781.7b1eed3780f4.59255.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:231b985f36d43f817df84e527d0a9e2a2c51f5fba78f1312ad0ce84cd167bd34
3
+ size 4357
experiments/logs/events.out.tfevents.1677007927.7b1eed3780f4.62309.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9434bfc711ed73659875ea734d5ef6c0a3c4da8a9988dbfda3bc8d15d6eb18f4
3
+ size 5280
experiments/logs/events.out.tfevents.1677008561.7b1eed3780f4.3852.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e598b68031d0e4b024d334bbdd8f9578605d37afad9e12883d40569898022704
3
+ size 6357
experiments/logs/events.out.tfevents.1677009134.7b1eed3780f4.11652.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7beda0e65719bc42ea79ca8271a49e66236fe8a545d6df0f15e0bccf4d151cc6
3
+ size 196022
experiments/logs/train_neo/events.out.tfevents.1676989999.7b1eed3780f4.2829.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2b32ba109e1360ab5b662d33639c8269cff54e5c6b2b3db72ddbd367a99fac3
3
+ size 3282
experiments/logs/train_neo/events.out.tfevents.1676990139.7b1eed3780f4.5567.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6fe69eaf4075333c1258fd65cc71b802cdf1bb6cbe92a208bbbccb6da2e45670
3
+ size 11778
experiments/logs/train_neo/events.out.tfevents.1676990697.7b1eed3780f4.10223.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b135479273cad4c3731d46a46bbd33836789a12fbc8410939f6db61548beca21
3
+ size 341626
experiments/logs/train_neo/events.out.tfevents.1677006978.7b1eed3780f4.46131.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a7183fffe76771d6f41c9327c7a4e712703dd37770facb441f9ec4187e0f4d2
3
+ size 4049
experiments/logs/train_neo/events.out.tfevents.1677007402.7b1eed3780f4.51624.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fc001b4583c619b0ac7b8b1d9f87ee06c44e21bb8b608157e4ace4e3f213985
3
+ size 2043
experiments/logs/train_neo/events.out.tfevents.1677007642.7b1eed3780f4.56432.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2fc5f10ff92471189468976c3bfcdd39a381332cd35e72beddaac389a968102
3
+ size 627
experiments/logs/train_neo/events.out.tfevents.1677007765.7b1eed3780f4.59255.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9b1c3b9a91703f47c9076edd408eba70052a2c5b5377145092942f48daa0943
3
+ size 863
experiments/logs/train_neo/events.out.tfevents.1677007911.7b1eed3780f4.62309.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5aed7d3a1b9b70a512390aa06f65ee6f4ee67bb22476ba9262c40aa1cd9900c
3
+ size 4167
experiments/logs/train_neo/events.out.tfevents.1677008546.7b1eed3780f4.3852.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:753aebddba05c1ac14d703f0ff956fd7046abe63675889cde20723c61e83e6db
3
+ size 8297
experiments/logs/train_neo/events.out.tfevents.1677009119.7b1eed3780f4.11652.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:060fae387dbb47c447acafec429237e7bfbdbb16d0fc159199e60e3f9fcf0155
3
+ size 732346
experiments/output.log ADDED
The diff for this file is too large to render. See raw diff
 
experiments/trainer_final_checkpoint/config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "EleutherAI/gpt-neo-125M",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPTNeoForCausalLM"
6
+ ],
7
+ "attention_dropout": 0,
8
+ "attention_layers": [
9
+ "global",
10
+ "local",
11
+ "global",
12
+ "local",
13
+ "global",
14
+ "local",
15
+ "global",
16
+ "local",
17
+ "global",
18
+ "local",
19
+ "global",
20
+ "local"
21
+ ],
22
+ "attention_types": [
23
+ [
24
+ [
25
+ "global",
26
+ "local"
27
+ ],
28
+ 6
29
+ ]
30
+ ],
31
+ "bos_token_id": 50256,
32
+ "embed_dropout": 0,
33
+ "eos_token_id": 50256,
34
+ "gradient_checkpointing": false,
35
+ "hidden_size": 768,
36
+ "initializer_range": 0.02,
37
+ "intermediate_size": null,
38
+ "layer_norm_epsilon": 1e-05,
39
+ "max_position_embeddings": 2048,
40
+ "model_type": "gpt_neo",
41
+ "num_heads": 12,
42
+ "num_layers": 12,
43
+ "resid_dropout": 0,
44
+ "summary_activation": null,
45
+ "summary_first_dropout": 0.1,
46
+ "summary_proj_to_labels": true,
47
+ "summary_type": "cls_index",
48
+ "summary_use_proj": true,
49
+ "torch_dtype": "float32",
50
+ "transformers_version": "4.24.0",
51
+ "use_cache": true,
52
+ "vocab_size": 50257,
53
+ "window_size": 256
54
+ }
experiments/trainer_final_checkpoint/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
experiments/trainer_final_checkpoint/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6589651dd8c9588a8faa97daec7814ffcfe998ba2e481eabdcb3e75405761d5d
3
+ size 551154684
experiments/trainer_final_checkpoint/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<|endoftext|>",
17
+ "unk_token": {
18
+ "content": "<|endoftext|>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
experiments/trainer_final_checkpoint/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
experiments/trainer_final_checkpoint/tokenizer_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "eos_token": {
13
+ "__type": "AddedToken",
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ },
20
+ "errors": "replace",
21
+ "model_max_length": 2048,
22
+ "name_or_path": "EleutherAI/gpt-neo-125M",
23
+ "pad_token": null,
24
+ "special_tokens_map_file": null,
25
+ "tokenizer_class": "GPT2Tokenizer",
26
+ "unk_token": {
27
+ "__type": "AddedToken",
28
+ "content": "<|endoftext|>",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false
33
+ }
34
+ }
experiments/trainer_final_checkpoint/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69255842ed78e689b179c20bae1499576b967ef499d84d9ec162b162d779e910
3
+ size 4475
experiments/trainer_final_checkpoint/vocab.json ADDED
The diff for this file is too large to render. See raw diff