Upload folder using huggingface_hub

Llama3_1Base-L17A-32x/checkpoints/final.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62a49b8ca5ed28e7a03e689892d3b201f7e48bded7c99f2174f6b508d1f56ce5
-size 2148278848
+oid sha256:51b3cdc15ff035003fa9316f6b290672facdfe0eda2f1489fb822e1340a8b1c4
+size 2147754376

Llama3_1Base-L17A-32x/hyperparams.json

@@ -1,4 +1,7 @@
 {
+    "device": "cuda:0",
+    "seed": 42,
+    "dtype": "torch.bfloat16",
     "hook_point_in": "blocks.17.hook_attn_out",
     "hook_point_out": "blocks.17.hook_attn_out",
     "use_decoder_bias": true,
@@ -6,17 +9,26 @@
     "expansion_factor": 32,
     "d_model": 4096,
     "d_sae": 131072,
-    "norm_activation": "token-wise",
-    "dataset_average_activation_norm": null,
+    "bias_init_method": "all_zero",
+    "act_fn": "jumprelu",
+    "jump_relu_threshold": 0.1259765625,
+    "norm_activation": "dataset-wise",
+    "dataset_average_activation_norm": {
+        "in": 3.046875,
+        "out": 3.046875
+    },
     "decoder_exactly_fixed_norm": false,
     "sparsity_include_decoder_norm": true,
     "use_glu_encoder": false,
-    "init_decoder_norm": null,
+    "init_decoder_norm": 0.5,
     "init_encoder_norm": null,
     "init_encoder_with_decoder_transpose": true,
-    "l1_coefficient": 1.6e-05,
-    "l1_coefficient_warmup_steps": 29296,
     "lp": 1,
+    "l1_coefficient": 8e-05,
+    "l1_coefficient_warmup_steps": 78125,
+    "top_k": 50,
+    "k_warmup_steps": 78125,
+    "use_batch_norm_mse": true,
     "use_ghost_grads": false,
     "tp_size": 1,
     "ddp_size": 1

Llama3_1Base-L17A-32x/lm_config.json

@@ -1,4 +1,7 @@
 {
+    "device": "cuda",
+    "seed": 42,
+    "dtype": "torch.bfloat16",
     "model_name": "meta-llama/Meta-Llama-3.1-8B",
     "use_flash_attn": false,
     "cache_dir": null,