Finnish-NLP
/

llama-3b-finnish

Text Generation

text-generation-inference

Model card Files Files and versions Community

aapot commited on Sep 7, 2023

Commit

b0df802

•

1 Parent(s): 440e354

Update

Files changed (1) hide show

pretrain_llama_3b.sh +2 -2

pretrain_llama_3b.sh CHANGED Viewed

@@ -8,7 +8,7 @@ export LIBTPU_INIT_ARGS='--xla_jf_spmd_threshold_for_windowed_einsum_mib=0 --xla
 python3 -m EasyLM.models.llama.llama_train \
-    --initialize_jax_distributed=True \
     --mesh_dim='1,-1,1' \
     --dtype='bf16' \
     --total_steps=1000000 \
@@ -20,7 +20,7 @@ python3 -m EasyLM.models.llama.llama_train \
     --update_llama_config='' \
     --load_dataset_state='' \
     --load_checkpoint='' \
-    --tokenizer.vocab_file='tokenizer.model' \
     --optimizer.type='lion' \
     --optimizer.lion_optimizer.weight_decay=1.0 \
     --optimizer.lion_optimizer.lr=3e-5 \

 python3 -m EasyLM.models.llama.llama_train \
+    --jax_distributed.initialize_jax_distributed=True \
     --mesh_dim='1,-1,1' \
     --dtype='bf16' \
     --total_steps=1000000 \
     --update_llama_config='' \
     --load_dataset_state='' \
     --load_checkpoint='' \
+    --tokenizer.pretrained_model_name_or_path='./' \
     --optimizer.type='lion' \
     --optimizer.lion_optimizer.weight_decay=1.0 \
     --optimizer.lion_optimizer.lr=3e-5 \