| base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T | |
| model_type: LlamaForCausalLM | |
| is_mistral_derived_model: false | |
| datasets: | |
| - path: burkelibbey/colors | |
| type: | |
| field_instruction: color | |
| field_output: description | |
| conversation: chatml | |
| chat_template: chatml | |
| val_set_size: 0.05 | |
| output_dir: ./out | |
| sequence_len: 512 | |
| gradient_accumulation_steps: 4 | |
| micro_batch_size: 2 | |
| learning_rate: 0.0002 | |
| load_in_4bit: true | |
| adapter: qlora | |
| lora_r: 16 | |
| lora_alpha: 8 | |
| lora_dropout: 0.05 | |
| lora_target_linear: true | |
| gradient_checkpointing: true | |
| warmup_steps: 10 | |
| tokenizer_type: LlamaTokenizer | |
| num_epochs: 1 | |
| optimizer: paged_adamw_32bit | |
| special_tokens: | |
| bos_token: "<s>" | |
| eos_token: "</s>" | |
| unk_token: "<unk>" | |