Hzfinfdu commited on
Commit
43acbce
1 Parent(s): 1fe0aae

Upload folder using huggingface_hub

Browse files
Llama3_1Base-L29R-32x/checkpoints/final.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ced537b993fea026954f3102b8d68077fb981a1f969969ee715c0023acf84edf
3
- size 2148278848
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9fd642fd590a16f3763cfd6e11ac3269fbc545742584dfc5d5a890f4a5b6073
3
+ size 2147754376
Llama3_1Base-L29R-32x/hyperparams.json CHANGED
@@ -1,4 +1,7 @@
1
  {
 
 
 
2
  "hook_point_in": "blocks.29.hook_resid_post",
3
  "hook_point_out": "blocks.29.hook_resid_post",
4
  "use_decoder_bias": true,
@@ -6,17 +9,26 @@
6
  "expansion_factor": 32,
7
  "d_model": 4096,
8
  "d_sae": 131072,
9
- "norm_activation": "token-wise",
10
- "dataset_average_activation_norm": null,
 
 
 
 
 
 
11
  "decoder_exactly_fixed_norm": false,
12
  "sparsity_include_decoder_norm": true,
13
  "use_glu_encoder": false,
14
- "init_decoder_norm": null,
15
  "init_encoder_norm": null,
16
  "init_encoder_with_decoder_transpose": true,
17
- "l1_coefficient": 1.6e-05,
18
- "l1_coefficient_warmup_steps": 29296,
19
  "lp": 1,
 
 
 
 
 
20
  "use_ghost_grads": false,
21
  "tp_size": 1,
22
  "ddp_size": 1
 
1
  {
2
+ "device": "cuda:0",
3
+ "seed": 42,
4
+ "dtype": "torch.bfloat16",
5
  "hook_point_in": "blocks.29.hook_resid_post",
6
  "hook_point_out": "blocks.29.hook_resid_post",
7
  "use_decoder_bias": true,
 
9
  "expansion_factor": 32,
10
  "d_model": 4096,
11
  "d_sae": 131072,
12
+ "bias_init_method": "all_zero",
13
+ "act_fn": "jumprelu",
14
+ "jump_relu_threshold": 1.1328125,
15
+ "norm_activation": "dataset-wise",
16
+ "dataset_average_activation_norm": {
17
+ "in": 46.5,
18
+ "out": 46.5
19
+ },
20
  "decoder_exactly_fixed_norm": false,
21
  "sparsity_include_decoder_norm": true,
22
  "use_glu_encoder": false,
23
+ "init_decoder_norm": 0.5,
24
  "init_encoder_norm": null,
25
  "init_encoder_with_decoder_transpose": true,
 
 
26
  "lp": 1,
27
+ "l1_coefficient": 8e-05,
28
+ "l1_coefficient_warmup_steps": 78125,
29
+ "top_k": 50,
30
+ "k_warmup_steps": 78125,
31
+ "use_batch_norm_mse": true,
32
  "use_ghost_grads": false,
33
  "tp_size": 1,
34
  "ddp_size": 1
Llama3_1Base-L29R-32x/lm_config.json CHANGED
@@ -1,4 +1,7 @@
1
  {
 
 
 
2
  "model_name": "meta-llama/Meta-Llama-3.1-8B",
3
  "use_flash_attn": false,
4
  "cache_dir": null,
 
1
  {
2
+ "device": "cuda",
3
+ "seed": 42,
4
+ "dtype": "torch.bfloat16",
5
  "model_name": "meta-llama/Meta-Llama-3.1-8B",
6
  "use_flash_attn": false,
7
  "cache_dir": null,