Hugging Face's logo

thoughtworks
/
arithmetic-sorl

Safetensors
sorl
arithmetic
interpretability
mechanistic-interpretability
qwen3
Model card Files
xet
 Community
amirali1985 commited on
Commit
d1932c1
·
verified ·
1 Parent(s): 8019738

Upload add_baseline_10K

Browse files
Files changed (5) hide show
  1. add_baseline_10K/config.json +38 -0
  2. add_baseline_10K/generation_config.json +7 -0
  3. add_baseline_10K/metrics.json +49 -0
  4. add_baseline_10K/model.safetensors +3 -0
  5. add_baseline_10K/train_config.json +28 -0
add_baseline_10K/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "SorlModelWrapper"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "float32",
9
+ "eos_token_id": null,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 512,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 2048,
15
+ "layer_types": [
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention"
19
+ ],
20
+ "max_position_embeddings": 128,
21
+ "max_window_layers": 28,
22
+ "model_type": "qwen3",
23
+ "num_attention_heads": 4,
24
+ "num_hidden_layers": 3,
25
+ "num_key_value_heads": 4,
26
+ "pad_token_id": null,
27
+ "rms_norm_eps": 1e-06,
28
+ "rope_parameters": {
29
+ "rope_theta": 10000.0,
30
+ "rope_type": "default"
31
+ },
32
+ "sliding_window": null,
33
+ "tie_word_embeddings": false,
34
+ "transformers_version": "5.5.0",
35
+ "use_cache": true,
36
+ "use_sliding_window": false,
37
+ "vocab_size": 151645
38
+ }
add_baseline_10K/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "output_attentions": false,
4
+ "output_hidden_states": false,
5
+ "transformers_version": "5.5.0",
6
+ "use_cache": true
7
+ }
add_baseline_10K/metrics.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "history": {
3
+ "step": [
4
+ 50,
5
+ 100,
6
+ 150,
7
+ 200,
8
+ 250,
9
+ 300,
10
+ 350,
11
+ 400,
12
+ 450
13
+ ],
14
+ "loss": [
15
+ 7.277637958526611,
16
+ 4.273232460021973,
17
+ 2.034315347671509,
18
+ 1.7297922372817993,
19
+ 1.0091018676757812,
20
+ 0.6293113827705383,
21
+ 0.5022323727607727,
22
+ 0.41768890619277954,
23
+ 0.35986801981925964
24
+ ],
25
+ "base_loss": [
26
+ 7.277637958526611,
27
+ 4.273232460021973,
28
+ 2.034315347671509,
29
+ 1.7297922372817993,
30
+ 1.0091018676757812,
31
+ 0.6293113827705383,
32
+ 0.5022323727607727,
33
+ 0.41768890619277954,
34
+ 0.35986801981925964
35
+ ],
36
+ "lr": [
37
+ 4.1702127659574475e-05,
38
+ 7.99652844364738e-05,
39
+ 7.587183774382011e-05,
40
+ 6.564050719821061e-05,
41
+ 5.102193404079467e-05,
42
+ 3.451744270264677e-05,
43
+ 1.89510493830239e-05,
44
+ 6.9862561552686355e-06,
45
+ 6.703099901399546e-07
46
+ ]
47
+ },
48
+ "final_accuracy": 0.41
49
+ }
add_baseline_10K/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f112563979930a3d6c47e0c74a5f47bbfdef8c98486f8129260cbc5444ad913
3
+ size 671794850
add_baseline_10K/train_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mode": "baseline",
3
+ "ops": "add",
4
+ "n_digits": 6,
5
+ "n_layer": 3,
6
+ "n_head": 4,
7
+ "n_embd": 512,
8
+ "abs_vocab": 0,
9
+ "K": 4,
10
+ "batch_size": 64,
11
+ "num_epochs": 3,
12
+ "dataset_size": 10000,
13
+ "lr": 8e-05,
14
+ "output_dir": "ckpt/r/add_baseline_10K",
15
+ "device": "cuda",
16
+ "push_to_hub": true,
17
+ "no_wandb": false,
18
+ "n_params": 167871744,
19
+ "run_name": "add_baseline_10K",
20
+ "git_commit": "9e4530548a98f8c7f5c14930ac4aec4886bb4b1b",
21
+ "timestamp": "2026-04-07T05:23:18.542015",
22
+ "tokenizer": "Qwen/Qwen3-0.6B",
23
+ "dataset_repo": "thoughtworks/arithmetic-sorl-data",
24
+ "dataset_config": "add_6digit",
25
+ "model_repo": "thoughtworks/arithmetic-sorl",
26
+ "trainer_version": "sft",
27
+ "final_accuracy": 0.41
28
+ }