Hzfinfdu commited on
Commit
47a9ac4
1 Parent(s): bbb7135

Rename folders

Browse files
Llama3_1Base-L0R-32x/checkpoints/final.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0d8d483a4a32318d0c1044fc2cb4ae7004e545d33fc186ceaceafd3689a21d09
3
- size 2147754376
 
 
 
 
Llama3_1Base-L0R-32x/hyperparams.json DELETED
@@ -1,35 +0,0 @@
1
- {
2
- "device": "cuda:0",
3
- "seed": 42,
4
- "dtype": "torch.bfloat16",
5
- "hook_point_in": "blocks.0.hook_resid_post",
6
- "hook_point_out": "blocks.0.hook_resid_post",
7
- "use_decoder_bias": true,
8
- "apply_decoder_bias_to_pre_encoder": false,
9
- "expansion_factor": 32,
10
- "d_model": 4096,
11
- "d_sae": 131072,
12
- "bias_init_method": "all_zero",
13
- "act_fn": "jumprelu",
14
- "jump_relu_threshold": 0.010986328125,
15
- "norm_activation": "dataset-wise",
16
- "dataset_average_activation_norm": {
17
- "in": 1.0859375,
18
- "out": 1.0859375
19
- },
20
- "decoder_exactly_fixed_norm": false,
21
- "sparsity_include_decoder_norm": true,
22
- "use_glu_encoder": false,
23
- "init_decoder_norm": 0.5,
24
- "init_encoder_norm": null,
25
- "init_encoder_with_decoder_transpose": true,
26
- "lp": 1,
27
- "l1_coefficient": 8e-05,
28
- "l1_coefficient_warmup_steps": 78125,
29
- "top_k": 50,
30
- "k_warmup_steps": 78125,
31
- "use_batch_norm_mse": true,
32
- "use_ghost_grads": false,
33
- "tp_size": 1,
34
- "ddp_size": 1
35
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Llama3_1Base-L0R-32x/lm_config.json DELETED
@@ -1,10 +0,0 @@
1
- {
2
- "device": "cuda",
3
- "seed": 42,
4
- "dtype": "torch.bfloat16",
5
- "model_name": "meta-llama/Meta-Llama-3.1-8B",
6
- "use_flash_attn": false,
7
- "cache_dir": null,
8
- "d_model": 4096,
9
- "local_files_only": false
10
- }