| mode: pt | |
| device: gpu | |
| precision: bf16 | |
| eval_only: false | |
| predict_only: false | |
| seed: 93789 | |
| tokenizer: | |
| name: BEE-spoke-data/hf_slimpajama-6B-28672-BPE-forT5 | |
| working_dir: null | |
| model: | |
| liger: true | |
| klass: local_t5 | |
| name: pszemraj/tFINE-850m-24x24-1024ctx | |
| overwrite: | |
| dropout_rate: 0.0 | |
| num_decoder_layers: 16 | |
| num_key_value_heads: 4 | |
| num_layers: 16 | |
| use_gqa: true | |
| add_config: | |
| is_bf16: false | |
| checkpoint_path: '' | |
| random_init: true | |
| compile: true | |
| data: | |
| multi_task: true | |
| NTP: 0.3 | |
| input_length: 512 | |
| max_seq_len: 512 | |
| mlm_probability: 0.15 | |
| mean_noise_span_length: 3.0 | |
| num_workers: 0 | |
| optim: | |
| name: adamwscale | |
| base_lr: 0.001 | |
| batch_size: 128 | |
| total_steps: 65536 | |
| epochs: -1 | |
| warmup_steps: 5000 | |
| lr_scheduler: cosine | |
| weight_decay: 0.01 | |
| grad_clip: 1.0 | |
| grad_acc: 16 | |
| final_cosine: 2.0e-05 | |
| eval: | |
| every_steps: 500 | |
| steps: 0 | |
| checkpoint: | |
| every_steps: 1500 | |
| logging: | |
| every_steps: 25 | |
| grad_l2: true | |
| weights_l2: true | |
| use_wandb: true | |
| wandb_config: | |
| project: nanoT5 | |
| entity: amazingvince | |
| tags: | |
| - gqa | |
| - large | |
| - e32-d16 | |
| - 512 ctx | |
| mode: online | |