added small model

configs/finetune.yml

@@ -0,0 +1,90 @@
+# GPT-2 pretraining setup
+{
+   # parallelism settings ( you will want to change these based on your cluster setup, ideally scheduling pipeline stages
+   # across the node boundaries )
+   "pipe-parallel-size": 1,
+   "model-parallel-size": 1,
+   "finetune": True,
+   "data-path": "/mnt/ssd-1/data/enron/enron_text_document",
+   "vocab-file": "/mnt/ssd-1/data/gpt2-vocab.json",
+   "merge-file": "/mnt/ssd-1/data/gpt2-merges.txt",
+   "load": "/mnt/ssd-1/neox_checkpoints/dense_small_checkpoints",
+   "save": "/mnt/ssd-1/neox_checkpoints/dense_small_checkpoints_finetune",
+
+   # model settings
+   "num-layers": 12,
+   "hidden-size": 768,
+   "num-attention-heads": 6,
+   "seq-length": 2048,
+   "max-position-embeddings": 2048,
+   "norm": "layernorm",
+   "pos-emb": "rotary",
+   "no-weight-tying": true,
+
+   # these should provide some speedup but takes a while to build, set to true if desired
+   "scaled-upper-triang-masked-softmax-fusion": false,
+   "bias-gelu-fusion": false,
+
+
+   # optimizer settings
+   "optimizer": {
+     "type": "Adam",
+     "params": {
+       "lr": 0.0006,
+       "betas": [0.9, 0.999],
+       "eps": 1.0e-8,
+     }
+   },
+   "zero_optimization": {
+    "stage": 1,
+    "allgather_partitions": True,
+    "allgather_bucket_size": 500000000,
+    "overlap_comm": True,
+    "reduce_scatter": True,
+    "reduce_bucket_size": 500000000,
+    "contiguous_gradients": True,
+    "cpu_offload": False
+  },
+
+   # batch / data settings
+   "train_micro_batch_size_per_gpu": 4,
+   "data-impl": "mmap",
+   "split": "949,50,1",
+
+   # activation checkpointing
+   "checkpoint-activations": true,
+   "checkpoint-num-layers": 1,
+   "partition-activations": true,
+   "synchronize-each-layer": true,
+
+   # regularization
+   "gradient_clipping": 1.0,
+   "weight-decay": 0.0,
+   "hidden-dropout": 0.0,
+   "attention-dropout": 0.0,
+
+   # precision settings
+   "fp16": { 
+     "enabled": true,
+     "loss_scale": 0,
+     "loss_scale_window": 1000,
+     "hysteresis": 2,
+     "min_loss_scale": 1
+   },
+
+   # misc. training settings
+   "train-iters": 320000,
+   "lr-decay-iters": 320000,
+   "distributed-backend": "nccl",
+   "lr-decay-style": "cosine",
+   "warmup": 0.01,
+   "save-interval": 1000,
+   "eval-interval": 1000,
+   "eval-iters": 10,
+
+   # logging
+   "log-interval": 100,
+   "steps_per_print": 10,
+   "keep-last-n-checkpoints": 4,
+   "wall_clock_breakdown": true,
+}

configs/sampling.yml

@@ -0,0 +1,27 @@
+# Data paths and options when using EleutherAI cluster
+{
+  "data-path": "/mnt/ssd-1/data/enron/enron_text_document",
+  # or for weighted datasets: 
+  # "train-data-paths": ["/mnt/ssd-1/data/enron/enron_text_document", "/mnt/ssd-cluster/data/enron/enron_text_document"],
+  # "test-data-paths": ["/mnt/ssd-1/data/enron/enron_text_document", "/mnt/ssd-cluster/data/enron/enron_text_document"],
+  # "valid-data-paths": ["/mnt/ssd-1/data/enron/enron_text_document", "/mnt/ssd-cluster/data/enron/enron_text_document"],
+  # "train-data-weights": [1., 2.],
+  # "test-data-weights": [2., 1.],
+  # "valid-data-weights": [0.5, 0.4],
+
+  "vocab-file": "/mnt/ssd-1/data/gpt2-vocab.json",
+  "merge-file": "/mnt/ssd-1/data/gpt2-merges.txt",
+  "save": "/mnt/ssd-1/neox_checkpoints/dense_small_checkpoints",
+  "load": "/mnt/ssd-1/neox_checkpoints/dense_small_checkpoints",
+  "tensorboard-dir": "/mnt/ssd-1/tensorboard",
+  "log-dir": "/mnt/ssd-1/logs",
+  "wandb_team": "eleutherai",
+  "sample-input-file":"/home/mchorse/gpt-neox/samplefile.txt",
+  "sample-output-file":"/home/mchorse/gpt-neox/sampleoutput.txt",
+  "text-gen-type": "unconditional",
+  "maximum_tokens": 256,
+  "temperature": 1.0,
+  "top_p": 0.0,
+  "top_k": 0,
+  "recompute": false,
+}

configs/small.yml

@@ -0,0 +1,84 @@
+# GPT-2 pretraining setup
+{
+   # parallelism settings ( you will want to change these based on your cluster setup, ideally scheduling pipeline stages
+   # across the node boundaries )
+   "pipe-parallel-size": 1,
+   "model-parallel-size": 1,
+
+   # model settings
+   "num-layers": 12,
+   "hidden-size": 768,
+   "num-attention-heads": 6,
+   "seq-length": 2048,
+   "max-position-embeddings": 2048,
+   "norm": "layernorm",
+   "pos-emb": "rotary",
+   "no-weight-tying": true,
+
+   # these should provide some speedup but takes a while to build, set to true if desired
+   "scaled-upper-triang-masked-softmax-fusion": false,
+   "bias-gelu-fusion": false,
+
+
+   # optimizer settings
+   "optimizer": {
+     "type": "Adam",
+     "params": {
+       "lr": 0.0006,
+       "betas": [0.9, 0.999],
+       "eps": 1e-8,
+     }
+   },
+   "zero_optimization": {
+    "stage": 0,
+    "allgather_partitions": True,
+    "allgather_bucket_size": 500000000,
+    "overlap_comm": True,
+    "reduce_scatter": True,
+    "reduce_bucket_size": 500000000,
+    "contiguous_gradients": True,
+    "cpu_offload": False
+  },
+
+   # batch / data settings
+   "train_micro_batch_size_per_gpu": 4,
+   "data-impl": "mmap",
+   "split": "949,50,1",
+
+   # activation checkpointing
+   "checkpoint-activations": true,
+   "checkpoint-num-layers": 1,
+   "partition-activations": true,
+   "synchronize-each-layer": true,
+
+   # regularization
+   "gradient_clipping": 1.0,
+   "weight-decay": 0.0,
+   "hidden-dropout": 0.0,
+   "attention-dropout": 0.0,
+
+   # precision settings
+   "fp16": { 
+     "enabled": true,
+     "loss_scale": 0,
+     "loss_scale_window": 1000,
+     "hysteresis": 2,
+     "min_loss_scale": 1
+   },
+
+   # misc. training settings
+   "train-iters": 320000,
+   "lr-decay-iters": 320000,
+   "distributed-backend": "nccl",
+   "lr-decay-style": "cosine",
+   "warmup": 0.01,
+   "save-interval": 10000,
+   "eval-interval": 1000,
+   "eval-iters": 10,
+
+   # logging
+   "log-interval": 100,
+   "steps_per_print": 10,
+   "keep-last-n-checkpoints": 4,
+   "wall_clock_breakdown": true,
+}

global_step485000/layer_00-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109b7fd7c42f2c20a78d072734a221a4044080976796fcba411151e7c0406d4f
+size 324404148

global_step485000/layer_02-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0bf1cda644b54e2189957d52f3987feb56b9e4b510c3264320f1fcc0949e677
+size 324648957

global_step485000/layer_03-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e72643538750406217730262b36104e8983991f388c483a59c4bdda057ef09ee
+size 324648958

global_step485000/layer_04-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10724d5a2807b27609753bc1ffee946566e9af4c58b8577f1f43eec1818b69a1
+size 324648958

global_step485000/layer_05-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de342a452f07a34f16d76f421deffc5c64555c0ca18e99e57ddc8ceddd2a6d5
+size 324648958

global_step485000/layer_06-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ed3d8a53bb255ed56139bc1f4d5163cd98be27dbb55730bd8c3e935e05a0d5b
+size 324648958

global_step485000/layer_07-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5258cb95e086a0f3b210ed6380a6a006373a91d38a59f7f24feee16e7d7f993
+size 324648958

global_step485000/layer_08-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04e3f597f694b1b06488a75a1f0221ba067a563a7edf5751ebf55366c2cd83bc
+size 324648958

global_step485000/layer_09-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:988373b46cd96bafdbcaf76a83f8b4cb834c027a882be5cf407037c232711510
+size 324648958

global_step485000/layer_10-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5de602ebedb3c093e68fd59a7172eea63aac05b9d013f805a68d69f8436c401e
+size 324648958

global_step485000/layer_11-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c8317e1f516158a4c926538dfdc8d4012fec2393896bc371a0a59feb7257ab9
+size 324648958

global_step485000/layer_12-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:747295d9da87dcb6a9067fd67ce040c2267f2e7037c14ee2fc7cc192c357d50b
+size 324648958

global_step485000/layer_13-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0433a7b7b227eb296f480ca60ca639367b8b711edf6a82d54b534313a643d1d4
+size 324648958

global_step485000/layer_15-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8c5835513069320c0f2709b1d0f353b95523ae8e53a942b8b060d4e19c4a802
+size 243636

global_step485000/layer_16-model_00-model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ed58c6117923e69dbd3810a638179b3434ac5361166b21e207b4ed3a64bb88a
+size 324404084

global_step485000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bfd97934543effe4a01b91f1e50680e90156e4a9453f81f09d641a3d41554cf
+size 1947893660