run_config.json

{
  "meta.run_name": "LoveLace-M-nlu-sft_nfgN",
  "meta.validate": true,
  "meta.tags": [
    "nlu_sweep_100B_3B"
  ],
  "finetune.checkpoint": "LoveLace-M-vocab_X9Ge",
  "finetune.trainable_backbone": true,
  "finetune.mode": "sft",
  "model.recompute_kv": false,
  "model.trainable_embeddings": true,
  "model.dropout_att_mat": 0.0,
  "model.dropout_att_out": 0.0,
  "model.dropout_ffn_int": 0.0,
  "model.dropout_ffn_out": 0.0,
  "train.batch_size": 128,
  "train.batch_size_step": 16,
  "train.batches_per_epoch": 512,
  "train.length_sequence": 2048,
  "train.length_cache": 2048,
  "train.lr_cooldown_tokens": 1500000000,
  "train.lr_warmup_steps": 200,
  "train.lr_max": 3e-05,
  "train.lr_cooldown_ratio": 0.0,
  "train.opt_max_grad_norm": 1.0,
  "train.opt_weight_decay": 0.5,
  "train.opt_decay_init": true,
  "train.opt_decay_mask": [
    "embedding.weight"
  ],
  "finetune.sft_mix_name": "full_mix_equal",
  "finetune.sft_mix": [
    [
      "glue/cola",
      5,
      false
    ],
    [
      "glue/mnli",
      5,
      false
    ],
    [
      "glue/mrpc",
      5,
      false
    ],
    [
      "glue/qnli",
      5,
      false
    ],
    [
      "glue/qqp",
      5,
      false
    ],
    [
      "glue/rte",
      5,
      false
    ],
    [
      "glue/sst2",
      5,
      false
    ],
    [
      "glue/stsb",
      5,
      false
    ],
    [
      "mmlu/all",
      5,
      false
    ],
    [
      "race/all",
      5,
      false
    ],
    [
      "hellaswag/choice",
      5,
      false
    ],
    [
      "hellaswag/no_choice",
      5,
      false
    ],
    [
      "squad/v2",
      5,
      false
    ],
    [
      "obqa/main",
      5,
      false
    ],
    [
      "winogrande/no_choice",
      5,
      false
    ],
    [
      "arc/challenge",
      5,
      false
    ],
    [
      "arc/easy",
      5,
      false
    ],
    [
      "super_glue/boolq",
      5,
      false
    ],
    [
      "piqa/no_choice",
      5,
      false
    ],
    [
      "alpaca/cleaned",
      5,
      true
    ],
    [
      "orca/OpenOrca",
      5,
      true
    ],
    [
      "cnn_dailymail/3.0.0",
      5,
      false
    ],
    [
      "coqa/all",
      5,
      false
    ],
    [
      "tiny/instruct",
      5,
      false
    ],
    [
      "tiny/summary",
      5,
      false
    ]
  ],
  "model.architectures": [
    "LSWTForCausalLM"
  ],
  "model.bos_token_id": 2,
  "model.pad_token_id": 1,
  "model.eos_token_id": 2,
  "model._name_or_path": "./checkpoints/lswt_medium_2k4k_100B",
  "model.transformers_version": "4.37.2",
  "model.model_type": "lsw_transformer",
  "model.vocab_size": 50272,
  "model.d_vocab": 768,
  "model.d_model": 1536,
  "model.d_ffn": 4096,
  "model.n_heads": 24,
  "model.n_layers": 18,
  "model.n_registers": 0,
  "model.gated_ffn": true,
  "model.gated_att": false,
  "model.qk_norm": false,
  "model.init_std": 0.02,
  "model.enable_bias": true,
  "model.rope_base_freq": 500000,
  "model.rope_reversed": true,
  "model.rope_positions": 4096,
  "model.rope_dynamic": false,
  "model.rope_ntk_scale": 1.0,
  "model.rope_yarn_a": 0.07,
  "model.rope_yarn_b": 1.0,
  "model.dropout_layers": 0.0,
  "model.use_cache": true,
  "model.parent_embeddings": "facebook/opt-125m",
  "model.reward_heads": null,
  "model.reward_pooler": "identity",
  "model.reward_dropout": 0.0,
  "train.optimizer": "LaProp",
  "train.opt_beta_1": 0.9,
  "train.opt_beta_2": 0.95,
  "train.opt_eps": 1e-08,
  "train.opt_rho": 0.1,
  "train.loss_objective": "MLE",
  "train.loss_sim_margin": 0.5,
  "params.total": 550978560,
  "params.trainable": 550978560,
  "params.non_trainable": 0
}