config.yaml

wandb:
  log: True
  entity: "weblab-geniac-leaderboard"
  project: "leaderboard"
  run_name: 'weblab-geniac2/exp_012_dMoE_8B_finetune_all_v6_epoch2_v0.1' # 変更可：weblab-geniac7/exp_{num}_{model_name}にします

github_version: v2.0.0 #for recording

testmode: false

# if you don't use api, please set "api" as "false"
# if you use api, please select from "openai", "anthoropic", "google", "cohere"
api: false

model:
  use_wandb_artifacts: false
  artifacts_path: ""
  pretrained_model_name_or_path: "geniacllm/dMoE_8B_finetune_all_v6_epoch2_v0.1" # 変更可
  trust_remote_code: true
  device_map: "auto"
  load_in_8bit: false
  load_in_4bit: false

generator:
  do_sample: false
  num_beams: 1 # https://huggingface.co/docs/transformers/v4.40.2/en/main_classes/text_generation
  top_p: 1.0
  top_k: 0
  temperature: 0.1
  repetition_penalty: 1.0

tokenizer:
  use_wandb_artifacts: false
  artifacts_path: ""
  pretrained_model_name_or_path: "geniacllm/dMoE_8B_finetune_all_v6_epoch2_v0.1" # 変更可
  use_fast: false

# for llm-jp-eval
max_seq_length: 2048 # 変更可
dataset_artifact: "wandb-japan/llm-leaderboard/jaster:v11" #if you use artifacts, please fill here (if not, fill null)
dataset_dir: "/jaster/1.2.6/evaluation/test"
target_dataset: "all" # {all, jamp, janli, jcommonsenseqa, jemhopqa, jnli, jsem, jsick, jsquad, jsts, niilc, chabsa}
log_dir: "./logs"
torch_dtype: "bf16" # {fp16, bf16, fp32}
custom_prompt_template: "<s>以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n### 指示:\n{instruction}\n\n### 入力:\n{input}\n\n### 応答:\n" # 変更可

custom_fewshots_template: null
# Please include {input} and {output} as variables
# example of fewshots template
# "\n### 入力：\n{input}\n### 回答：\n{output}"

metainfo:
  basemodel_name: "geniacllm/dMoE_8B_finetune_all_v6_epoch2_v0.1" # 変更可
  model_type: "open llm" # {open llm, commercial api}
  instruction_tuning_method: "None" # {"None", "Full", "LoRA", ...}
  instruction_tuning_data: ["None"] # {"None", "jaster", "dolly_ja", "oasst_ja", ...}
  num_few_shots: 0
  llm-jp-eval-version: "1.1.0"

# for mtbench
mtbench:
  question_artifacts_path: "wandb-japan/llm-leaderboard/mtbench_ja_question:v0" # if testmode is true, small dataset will be used
  referenceanswer_artifacts_path: "wandb-japan/llm-leaderboard/mtbench_ja_referenceanswer:v0" # if testmode is true, small dataset will be used
  judge_prompt_artifacts_path: "wandb-japan/llm-leaderboard/mtbench_ja_prompt:v1"
  bench_name: "japanese_mt_bench"
  model_id: null # cannot use '<', '>', ':', '"', '/', '\\', '|', '?', '*', '.'
  question_begin: null
  question_end: null
  max_new_token: 1024 # 変更可
  num_choices: 1
  num_gpus_per_model: 1 # 変更可
  num_gpus_total: 1 # 変更可
  max_gpu_memory: null
  dtype: bfloat16 # None or float32 or float16 or bfloat16
  # for gen_judgment
  judge_model: "gpt-4"
  mode: "single"
  baseline_model: null
  parallel: 1
  first_n: null
  # for conv template # added
  custom_conv_template: true # 変更可
  # the following variables will be used when custom_conv_template is set as true
  conv_name: "custom"
  conv_system_message: "<s>以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。" # 変更可
  conv_roles: "('指示', '応答')" # 変更可
  conv_sep: "\n\n### " # 変更可
  conv_stop_token_ids: "[2]"
  conv_stop_str: "###"
  conv_role_message_separator: ":\n" # 変更可（llm-jp-evalに合わせて改行前のスペースは削除した）
  conv_role_only_separator: ":\n" # 変更可（llm-jp-evalに合わせて改行前のスペースは削除した）