| act_frequency_n_tokens: 500000 | |
| batch_size: 8 | |
| collect_act_frequency_every_n_samples: 40000 | |
| collect_output_metrics_every_n_samples: 0 | |
| cooldown_samples: 0 | |
| effective_batch_size: 16 | |
| eval_data: | |
| column_name: input_ids | |
| dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2 | |
| is_tokenized: true | |
| n_ctx: 1024 | |
| seed: 0 | |
| split: train | |
| streaming: true | |
| tokenizer_name: gpt2 | |
| eval_every_n_samples: 40000 | |
| eval_n_samples: 500 | |
| log_every_n_grad_steps: 20 | |
| loss: | |
| in_to_orig: null | |
| logits_kl: null | |
| out_to_in: | |
| coeff: 1.0 | |
| out_to_orig: null | |
| sparsity: | |
| coeff: 4.0 | |
| p_norm: 1.0 | |
| lr: 0.0005 | |
| lr_schedule: cosine | |
| max_grad_norm: 10.0 | |
| min_lr_factor: 0.1 | |
| n_samples: 400000 | |
| saes: | |
| dict_size_to_input_ratio: 60.0 | |
| k: null | |
| pretrained_sae_paths: null | |
| retrain_saes: false | |
| sae_positions: | |
| - blocks.2.hook_resid_pre | |
| type_of_sparsifier: sae | |
| save_dir: /mnt/ssd-interp/dan/sparsify/sparsify/scripts/train_tlens_saes/out | |
| save_every_n_samples: null | |
| seed: 0 | |
| tlens_model_name: gpt2-small | |
| tlens_model_path: null | |
| train_data: | |
| column_name: input_ids | |
| dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2 | |
| is_tokenized: true | |
| n_ctx: 1024 | |
| seed: 0 | |
| split: train | |
| streaming: true | |
| tokenizer_name: gpt2 | |
| wandb_project: gpt2-layerwise_play | |
| wandb_run_name: null | |
| wandb_run_name_prefix: '' | |
| warmup_samples: 20000 | |