tinyllamas

https://github.com/ellishg/llama2.c

Training Tokenizers

python tinystories.py download
python tinystories.py train_vocab --vocab_size=<vocab_size>
python tinystories.py pretokenize --vocab_size=<vocab_size>
python tokenizer.py --tokenizer-model=data/tok<vocab_size>.model

Training Models

stories3_5M-v4k

python train.py --out_dir="out_3_5m_v4k" --batch_size=16 --max_seq_len=256 --gradient_accumulation_steps=8 --vocab_source="custom" --vocab_size=4096 --dim=208 --n_layers=6 --n_heads=8 --n_kv_heads=4 --multiple_of=4 --learning_rate=1e-3 --dropout=0.05 --weight_decay=0.01 --max_iters=100000 --beta2=0.99 --warmup_iters=1000 --eval_interval=100 --eval_iters=100 --compile=False --device="mps"

python export.py --checkpoint stories3_5M-v4k.pt --version 0 stories3_5M-v4k.bin

stories3_5M-v32k

Trained to a loss of 1.3386

python train.py --out_dir="out_3_5m_v32k" --batch_size=16 --max_seq_len=512 --gradient_accumulation_steps=8 --vocab_source="custom" --vocab_size=32000 --dim=256 --n_layers=8 --n_heads=8 --n_kv_heads=4 --multiple_of=4 --learning_rate=1e-3 --dropout=0.05 --weight_decay=0.01 --max_iters=100000 --beta2=0.99 --warmup_iters=1000 --eval_interval=100 --eval_iters=100 --compile=False --device="mps"

python export.py --checkpoint stories3_5M-v32k.pt --version 0 stories3_5M-v32k.bin

int8 Quantized

python export.py --checkpoint stories3_5M-v32k.pt --version 2 stories3_5M-Q8_0-v32k.bin

Running Models

./run stories3_5M-v4k.bin -z tok4096.bin
./run stories3_5M-v32k.bin -z tok32000.bin
./runq stories3_5M-Q8_0-v32k.bin -z tok32000.bin