axolotl-config.yml

base_model: MediaTek-Research/Breeze-7B-Base-v0_1
model_type: MistralForCausalLM
tokenizer_type: LlamaTokenizer
is_mistral_derived_model: true

load_in_8bit: false
load_in_4bit: true
strict: false


datasets:
- path: hon9kon9ize/yue-alpaca
  type: alpaca
- path: indiejoseph/wikipedia-translate-zhhk-zhcn
  type:
    system_prompt: ""
    field_instruction: zh
    field_output: yue
    format: |-
      [INST]
      翻譯下面中文至粵語廣東話(Cantonese)。

      {instruction}
      [/INST]
- path: indiejoseph/wikipedia-zh-yue-summaries
  type:
    system_prompt: ""
    field_instruction: content
    field_output: summary
    format: |-
      [INST]
      用粵語廣東話(Cantonese)總結一吓。

      {instruction}
      [/INST]
- path: indiejoseph/wikipedia-zh-yue-summaries
  type:
    system_prompt: ""
    field_instruction: content
    field_output: title
    format: |-
      [INST]
      粵語廣東話(Cantonese), 呢篇嘢主題係咩?

      {instruction}
      [/INST]
- path: indiejoseph/wikipedia-zh-yue-summaries
  type:
    system_prompt: ""
    field_instruction: content
    field_output: category
    format: |-
      [INST]
      粵語廣東話(Cantonese), 呢篇嘢講緊咩? 係咩分類?

      {instruction}
      [/INST]
- path: indiejoseph/wikipedia-zh-yue-qa
  type:
    system_prompt: ""
    field_instruction: question
    field_system: title
    field_output: answer
    format: |-
      [INST]
      粵語廣東話(Cantonese), 以下係關於「{system}」嘅問題。

      {instruction}
      [/INST]

dataset_prepared_path: last_run_prepared
val_set_size: 0.05
output_dir: loras/Breeze-7B-Cantonese-v0.1
save_safetensors: true
#eval_sample_packing: False

## You can optionally freeze the entire model and unfreeze a subset of parameters
unfrozen_parameters:
#  - lm_head.*
#  - model.embed_tokens.*
#  - model.layers.2[0-9]+.block_sparse_moe.gate.*
#  - model.layers.2[0-9]+.block_sparse_moe.experts.*
#  - model.layers.3[0-9]+.block_sparse_moe.gate.*
#  - model.layers.3[0-9]+.block_sparse_moe.experts.*

model_config:
  output_router_logits: true

adapter: qlora
lora_model_dir:

sequence_len: 4096
sample_packing: true
pad_to_sequence_len: true

lora_r: 32
lora_alpha: 16
lora_dropout: 0.05
lora_target_linear: true
lora_fan_in_fan_out:


wandb_project:
wandb_entity:
wandb_watch:
wandb_name:
wandb_log_model:

gradient_accumulation_steps: 4
micro_batch_size: 2
num_epochs: 1
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
learning_rate: 0.0002

train_on_inputs: false
group_by_length: false
bf16: true
fp16: false
tf32: false

gradient_checkpointing: true
early_stopping_patience:
resume_from_checkpoint:
local_rank:
logging_steps: 1
xformers_attention:
flash_attention: true

loss_watchdog_threshold: 5.0
loss_watchdog_patience: 3

warmup_steps: 10
evals_per_epoch: 4
eval_table_size:
eval_table_max_new_tokens: 128
saves_per_epoch: 1
debug:
deepspeed:
weight_decay: 0.0
fsdp:
fsdp_config:
special_tokens: