stellaathena
/

test-med

Model card Files Files and versions Community

test-med / configs /medium.yml

stellaathena's picture

Added model weights

360abcb about 2 years ago

raw history blame contribute delete

No virus

2.06 kB

	# GPT-2 pretraining setup
	{
	# parallelism settings ( you will want to change these based on your cluster setup, ideally scheduling pipeline stages
	# across the node boundaries )
	"pipe-parallel-size": 1,
	"model-parallel-size": 1,

	# model settings
	"num-layers": 24,
	"hidden-size": 1024,
	"num-attention-heads": 16,
	"seq-length": 2048,
	"max-position-embeddings": 2048,
	"norm": "layernorm",
	"pos-emb": "rotary",
	"no-weight-tying": true,

	# these should provide some speedup but takes a while to build, set to true if desired
	"scaled-upper-triang-masked-softmax-fusion": false,
	"bias-gelu-fusion": false,



	# optimizer settings
	"optimizer": {
	"type": "Adam",
	"params": {
	"lr": 0.0003,
	"betas": [0.9, 0.999],
	"eps": 1.0e-8,
	}
	},
	"zero_optimization": {
	"stage": 1,
	"allgather_partitions": True,
	"allgather_bucket_size": 500000000,
	"overlap_comm": True,
	"reduce_scatter": True,
	"reduce_bucket_size": 500000000,
	"contiguous_gradients": True,
	"cpu_offload": False
	},
	# batch / data settings
	"train_micro_batch_size_per_gpu": 4,
	"data-impl": "mmap",
	"split": "949,50,1",

	# activation checkpointing
	"checkpoint-activations": true,
	"checkpoint-num-layers": 1,
	"partition-activations": true,
	"synchronize-each-layer": true,

	# regularization
	"gradient_clipping": 1.0,
	"weight-decay": 0,
	"hidden-dropout": 0,
	"attention-dropout": 0,

	# precision settings
	"fp16": {
	"fp16": true,
	"enabled": true,
	"loss_scale": 0,
	"loss_scale_window": 1000,
	"hysteresis": 2,
	"min_loss_scale": 1
	},

	# misc. training settings
	"train-iters": 320000,
	"lr-decay-iters": 320000,
	"distributed-backend": "nccl",
	"lr-decay-style": "cosine",
	"warmup": 0.01,
	"save-interval": 10000,
	"eval-interval": 1000,
	"eval-iters": 10,

	# logging
	"log-interval": 100,
	"steps_per_print": 10,
	"keep-last-n-checkpoints": 4,
	"wall_clock_breakdown": true,
	}