Upload SAEs (#1)

- Upload SAEs (89f993710a47a2e37034f91137a8360dca0e9bc3)

downstream_similar_ce_layer_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:071ebefad984390288ff036dc2952ffad150ca9be411be06667cb58b7d19c536
+size 283305160

downstream_similar_ce_layer_10.yaml

@@ -0,0 +1,62 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig:
+    hook_positions:
+    - blocks.11.hook_resid_pre
+    total_coeff: 0.05
+  logits_kl:
+    coeff: 0.5
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 1.75
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.10.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/dan_braun/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

downstream_similar_ce_layer_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbe566a12b302d22db01017fdc0059f2e3479c84654c03c2866623ce73002fdd
+size 283305152

downstream_similar_ce_layer_2.yaml

@@ -0,0 +1,60 @@
+act_frequency_n_tokens: 500000
+batch_size: 4
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 200
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig:
+    hook_positions:
+    - blocks.3.hook_resid_pre
+    - blocks.4.hook_resid_pre
+    - blocks.5.hook_resid_pre
+    - blocks.6.hook_resid_pre
+    - blocks.7.hook_resid_pre
+    - blocks.8.hook_resid_pre
+    - blocks.9.hook_resid_pre
+    - blocks.10.hook_resid_pre
+    - blocks.11.hook_resid_pre
+    total_coeff: 2.5
+  logits_kl:
+    coeff: 0.5
+  out_to_in:
+    coeff: 0
+  out_to_orig: null
+  sparsity:
+    coeff: 10
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60
+  sae_positions: blocks.2.hook_resid_pre
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e
+wandb_run_name: null
+wandb_run_name_prefix: recon_
+warmup_samples: 20000

downstream_similar_ce_layer_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68a2aaad18bf2855fb6086ce68bdf6e54a6911a89918ae86b41fff2daee17712
+size 283305152

downstream_similar_ce_layer_6.yaml

@@ -0,0 +1,66 @@
+act_frequency_n_tokens: 500000
+batch_size: 4
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 200
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig:
+    hook_positions:
+    - blocks.7.hook_resid_pre
+    - blocks.8.hook_resid_pre
+    - blocks.9.hook_resid_pre
+    - blocks.10.hook_resid_pre
+    - blocks.11.hook_resid_pre
+    total_coeff: 2.5
+  logits_kl:
+    coeff: 0.5
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 50.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.6.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e
+wandb_run_name: null
+wandb_run_name_prefix: recon_
+warmup_samples: 20000

downstream_similar_l0_layer_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cf449615184314b005e2eb0e5378e4ae5cc065f42bc3e748bb988476cabd447
+size 283305160

downstream_similar_l0_layer_10.yaml

@@ -0,0 +1,62 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig:
+    hook_positions:
+    - blocks.11.hook_resid_pre
+    total_coeff: 0.05
+  logits_kl:
+    coeff: 0.5
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 1.75
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.10.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/dan_braun/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

downstream_similar_l0_layer_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99744a3807d0e2287c62c680a56108f5a82a46c0b4b6e02344847b04608a792b
+size 283305152

downstream_similar_l0_layer_2.yaml

@@ -0,0 +1,60 @@
+act_frequency_n_tokens: 500000
+batch_size: 4
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 200
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig:
+    hook_positions:
+    - blocks.3.hook_resid_pre
+    - blocks.4.hook_resid_pre
+    - blocks.5.hook_resid_pre
+    - blocks.6.hook_resid_pre
+    - blocks.7.hook_resid_pre
+    - blocks.8.hook_resid_pre
+    - blocks.9.hook_resid_pre
+    - blocks.10.hook_resid_pre
+    - blocks.11.hook_resid_pre
+    total_coeff: 2.5
+  logits_kl:
+    coeff: 0.5
+  out_to_in:
+    coeff: 0
+  out_to_orig: null
+  sparsity:
+    coeff: 35
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60
+  sae_positions: blocks.2.hook_resid_pre
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e
+wandb_run_name: null
+wandb_run_name_prefix: recon_
+warmup_samples: 20000

downstream_similar_l0_layer_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76e7e12a4666688728574c762a20d92a82c0a6a403f77ef15968349a89548e22
+size 283305152

downstream_similar_l0_layer_6.yaml

@@ -0,0 +1,66 @@
+act_frequency_n_tokens: 500000
+batch_size: 4
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 200
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig:
+    hook_positions:
+    - blocks.7.hook_resid_pre
+    - blocks.8.hook_resid_pre
+    - blocks.9.hook_resid_pre
+    - blocks.10.hook_resid_pre
+    - blocks.11.hook_resid_pre
+    total_coeff: 2.5
+  logits_kl:
+    coeff: 0.5
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 50.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.6.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: null
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e
+wandb_run_name: null
+wandb_run_name_prefix: recon_
+warmup_samples: 20000

e2e_similar_ce_layer_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f98c3f3f7f21431e4025515987c05ad1edf4a68b78935d7748cf81a0486a85cb
+size 283305104

e2e_similar_ce_layer_10.yaml

@@ -0,0 +1,59 @@
+act_frequency_n_tokens: 500000
+batch_size: 16
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl:
+    coeff: 1.0
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 1.5
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.10.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

e2e_similar_ce_layer_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4864807ce659ebc967867707687982f6c3c4f3b61f4f880de5427a26bf7001bf
+size 283305096

e2e_similar_ce_layer_2.yaml

@@ -0,0 +1,59 @@
+act_frequency_n_tokens: 500000
+batch_size: 16
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl:
+    coeff: 1.0
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 0.5
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.2.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

e2e_similar_ce_layer_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:252358bb0c5bb9a5f18ad8c9e1cf4257948c6661af4f17ca24a162b1edb709b7
+size 283305096

e2e_similar_ce_layer_6.yaml

@@ -0,0 +1,59 @@
+act_frequency_n_tokens: 500000
+batch_size: 16
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl:
+    coeff: 1.0
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 3.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.6.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

e2e_similar_l0_layer_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d8369abe8f4e8962eff009b9fb7f681a046422dce32d01e1fe99725ab6eb86b
+size 283305104

e2e_similar_l0_layer_10.yaml

@@ -0,0 +1,59 @@
+act_frequency_n_tokens: 500000
+batch_size: 16
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl:
+    coeff: 1.0
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 1.5
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.10.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

e2e_similar_l0_layer_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44a7b30cff15bfac66eb46361627016ba1a6549ae15c8740c3119e33fcfd0c2d
+size 283305096

e2e_similar_l0_layer_2.yaml

@@ -0,0 +1,59 @@
+act_frequency_n_tokens: 500000
+batch_size: 16
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl:
+    coeff: 1.0
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 1.5
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.2.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

e2e_similar_l0_layer_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf5513c22c99845afcaf125ff203fdfac9d49477f99c59c0912aefe6c965d33f
+size 283305096

e2e_similar_l0_layer_6.yaml

@@ -0,0 +1,59 @@
+act_frequency_n_tokens: 500000
+batch_size: 16
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 20000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl:
+    coeff: 1.0
+  out_to_in:
+    coeff: 0.0
+  out_to_orig: null
+  sparsity:
+    coeff: 1.5
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.6.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /data/jordan_tensor/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-e2e_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

local_similar_ce_layer_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:353e4397a1910189a7317c5882804226a884fc5cf3bc03aa4b55bbbadfdaf161
+size 283305120

local_similar_ce_layer_10.yaml

@@ -0,0 +1,58 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl: null
+  out_to_in:
+    coeff: 1.0
+  out_to_orig: null
+  sparsity:
+    coeff: 6.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.10.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /mnt/ssd-interp/dan/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-layerwise_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

local_similar_ce_layer_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e197d179193f62de6633061aa82b74b2b8bec78a6844ceed8195f9539686c0
+size 283305112

local_similar_ce_layer_2.yaml

@@ -0,0 +1,58 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl: null
+  out_to_in:
+    coeff: 1.0
+  out_to_orig: null
+  sparsity:
+    coeff: 0.8
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.2.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /mnt/ssd-interp/dan/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-layerwise_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

local_similar_ce_layer_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49a66c6d9ae4f6ff636d887f31957caf1cffc6375078fcf9f7eb637a00b4ee4
+size 283305112

local_similar_ce_layer_6.yaml

@@ -0,0 +1,58 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl: null
+  out_to_in:
+    coeff: 1.0
+  out_to_orig: null
+  sparsity:
+    coeff: 4.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.6.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /mnt/ssd-interp/dan/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-layerwise_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

local_similar_l0_layer_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9f4178cf150c2402a9d71eae5322bc19d12af798d7f846971ed2ac6fc85506
+size 283305120

local_similar_l0_layer_10.yaml

@@ -0,0 +1,58 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl: null
+  out_to_in:
+    coeff: 1.0
+  out_to_orig: null
+  sparsity:
+    coeff: 10.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.10.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /mnt/ssd-interp/dan/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-layerwise_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

local_similar_l0_layer_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e24781476ae17f9c643177792075365b6f9c16963bc1e5d328730172e0394156
+size 283305112

local_similar_l0_layer_2.yaml

@@ -0,0 +1,58 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl: null
+  out_to_in:
+    coeff: 1.0
+  out_to_orig: null
+  sparsity:
+    coeff: 4.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.2.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /mnt/ssd-interp/dan/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-layerwise_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000

local_similar_l0_layer_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5517947946a84a2b26b787758da88897e15a6dd140c5a475e527da1aabe59dd7
+size 283305112

local_similar_l0_layer_6.yaml

@@ -0,0 +1,58 @@
+act_frequency_n_tokens: 500000
+batch_size: 8
+collect_act_frequency_every_n_samples: 40000
+collect_output_metrics_every_n_samples: 0
+cooldown_samples: 0
+effective_batch_size: 16
+eval_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+eval_every_n_samples: 40000
+eval_n_samples: 500
+log_every_n_grad_steps: 20
+loss:
+  in_to_orig: null
+  logits_kl: null
+  out_to_in:
+    coeff: 1.0
+  out_to_orig: null
+  sparsity:
+    coeff: 6.0
+    p_norm: 1.0
+lr: 0.0005
+lr_schedule: cosine
+max_grad_norm: 10.0
+min_lr_factor: 0.1
+n_samples: 400000
+saes:
+  dict_size_to_input_ratio: 60.0
+  k: null
+  pretrained_sae_paths: null
+  retrain_saes: false
+  sae_positions:
+  - blocks.6.hook_resid_pre
+  type_of_sparsifier: sae
+save_dir: /mnt/ssd-interp/dan/sparsify/sparsify/scripts/train_tlens_saes/out
+save_every_n_samples: null
+seed: 0
+tlens_model_name: gpt2-small
+tlens_model_path: null
+train_data:
+  column_name: input_ids
+  dataset_name: apollo-research/Skylion007-openwebtext-tokenizer-gpt2
+  is_tokenized: true
+  n_ctx: 1024
+  seed: 0
+  split: train
+  streaming: true
+  tokenizer_name: gpt2
+wandb_project: gpt2-layerwise_play
+wandb_run_name: null
+wandb_run_name_prefix: ''
+warmup_samples: 20000