Upload add_sub_baseline_10K_1L3H510d

add_sub_baseline_10K_1L3H510d/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151645
+}

add_sub_baseline_10K_1L3H510d/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_baseline_10K_1L3H510d/metrics.json

@@ -0,0 +1,831 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1600,
+      1650,
+      1700,
+      1750,
+      1800,
+      1850,
+      1900,
+      1950,
+      2000,
+      2050,
+      2100,
+      2150,
+      2200,
+      2250,
+      2300,
+      2350,
+      2400,
+      2450,
+      2500,
+      2550,
+      2600,
+      2650,
+      2700,
+      2750,
+      2800,
+      2850,
+      2900,
+      2950,
+      3000,
+      3050,
+      3100
+    ],
+    "loss": [
+      7.182804584503174,
+      4.20333194732666,
+      2.0677404403686523,
+      1.829144835472107,
+      1.8410581350326538,
+      1.820699691772461,
+      1.7079013586044312,
+      1.655383825302124,
+      1.6272556781768799,
+      1.56789231300354,
+      1.2935447692871094,
+      0.8198140263557434,
+      0.7896751761436462,
+      0.7217035293579102,
+      0.6449242830276489,
+      0.6155810356140137,
+      0.6273579001426697,
+      0.5691636800765991,
+      0.5195116400718689,
+      0.5188413262367249,
+      0.4571366012096405,
+      0.46536850929260254,
+      0.45766669511795044,
+      0.4275817275047302,
+      0.40487247705459595,
+      0.4464508593082428,
+      0.39323002099990845,
+      0.39669889211654663,
+      0.34110480546951294,
+      0.3717530369758606,
+      0.3865724503993988,
+      0.36133044958114624,
+      0.3357429802417755,
+      0.3121393620967865,
+      0.3670870363712311,
+      0.2978810667991638,
+      0.3120233118534088,
+      0.34596773982048035,
+      0.29005879163742065,
+      0.34079042077064514,
+      0.29599788784980774,
+      0.30136188864707947,
+      0.2756097614765167,
+      0.3011065125465393,
+      0.30379655957221985,
+      0.2766466736793518,
+      0.2690426707267761,
+      0.3143273890018463,
+      0.26247408986091614,
+      0.27483493089675903,
+      0.2900548279285431,
+      0.2740040123462677,
+      0.2515677511692047,
+      0.2737451195716858,
+      0.246206596493721,
+      0.2460457980632782,
+      0.2329309731721878,
+      0.25465384125709534,
+      0.2698599696159363,
+      0.2485286146402359,
+      0.24830453097820282,
+      0.2897026538848877
+    ],
+    "base_loss": [
+      7.182804584503174,
+      4.20333194732666,
+      2.0677404403686523,
+      1.829144835472107,
+      1.8410581350326538,
+      1.820699691772461,
+      1.7079013586044312,
+      1.655383825302124,
+      1.6272556781768799,
+      1.56789231300354,
+      1.2935447692871094,
+      0.8198140263557434,
+      0.7896751761436462,
+      0.7217035293579102,
+      0.6449242830276489,
+      0.6155810356140137,
+      0.6273579001426697,
+      0.5691636800765991,
+      0.5195116400718689,
+      0.5188413262367249,
+      0.4571366012096405,
+      0.46536850929260254,
+      0.45766669511795044,
+      0.4275817275047302,
+      0.40487247705459595,
+      0.4464508593082428,
+      0.39323002099990845,
+      0.39669889211654663,
+      0.34110480546951294,
+      0.3717530369758606,
+      0.3865724503993988,
+      0.36133044958114624,
+      0.3357429802417755,
+      0.3121393620967865,
+      0.3670870363712311,
+      0.2978810667991638,
+      0.3120233118534088,
+      0.34596773982048035,
+      0.29005879163742065,
+      0.34079042077064514,
+      0.29599788784980774,
+      0.30136188864707947,
+      0.2756097614765167,
+      0.3011065125465393,
+      0.30379655957221985,
+      0.2766466736793518,
+      0.2690426707267761,
+      0.3143273890018463,
+      0.26247408986091614,
+      0.27483493089675903,
+      0.2900548279285431,
+      0.2740040123462677,
+      0.2515677511692047,
+      0.2737451195716858,
+      0.246206596493721,
+      0.2460457980632782,
+      0.2329309731721878,
+      0.25465384125709534,
+      0.2698599696159363,
+      0.2485286146402359,
+      0.24830453097820282,
+      0.2897026538848877
+    ],
+    "lr": [
+      3.9200000000000004e-05,
+      7.92e-05,
+      7.994872780244471e-05,
+      7.979084217550451e-05,
+      7.952674320281786e-05,
+      7.915713584145437e-05,
+      7.868300668109943e-05,
+      7.810562131055899e-05,
+      7.742652093953451e-05,
+      7.664751828468545e-05,
+      7.57706927309605e-05,
+      7.47983847811137e-05,
+      7.373318980822093e-05,
+      7.25779511278734e-05,
+      7.133575240854014e-05,
+      7.000990944035905e-05,
+      6.860396128432721e-05,
+      6.712166082551651e-05,
+      6.556696475553043e-05,
+      6.394402301094192e-05,
+      6.225716769590408e-05,
+      6.0510901518502626e-05,
+      5.8709885771716645e-05,
+      5.685892789107026e-05,
+      5.4962968622187084e-05,
+      5.3027068832501364e-05,
+      5.1056396002328924e-05,
+      4.9056210431357356e-05,
+      4.703185119737419e-05,
+      4.498872190471344e-05,
+      4.293227626046202e-05,
+      4.086800351692724e-05,
+      3.880141381922381e-05,
+      3.673802349709187e-05,
+      3.468334034020644e-05,
+      3.2642848896282794e-05,
+      3.062199583122133e-05,
+      2.862617539037015e-05,
+      2.6660714999713274e-05,
+      2.4730861045419232e-05,
+      2.284176486970851e-05,
+      2.099846902042102e-05,
+      1.9205893790987304e-05,
+      1.7468824086732586e-05,
+      1.579189665257094e-05,
+      1.4179587696182778e-05,
+      1.2636200939713001e-05,
+      1.1165856131883247e-05,
+      9.772478051182794e-06,
+      8.459786029491775e-06,
+      7.231284024101261e-06,
+      6.090251264630804e-06,
+      5.039733499809587e-06,
+      4.082534867486105e-06,
+      3.221210409567612e-06,
+      2.458059251869167e-06,
+      1.7951184670772902e-06,
+      1.2341576372097185e-06,
+      7.766741300856728e-07,
+      4.23889102415056e-07,
+      1.7674424017557922e-07,
+      3.58992449786566e-08
+    ],
+    "eval_step": [
+      156,
+      312,
+      468,
+      624,
+      780,
+      936,
+      1092,
+      1248,
+      1404,
+      1560,
+      1716,
+      1872,
+      2028,
+      2184,
+      2340,
+      2496,
+      2652,
+      2808,
+      2964,
+      3120
+    ],
+    "eval_epoch": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0044444444444444444,
+      0.006666666666666667,
+      0.01888888888888889,
+      0.057777777777777775,
+      0.09444444444444444,
+      0.11888888888888889,
+      0.20222222222222222,
+      0.24555555555555555,
+      0.2611111111111111,
+      0.32666666666666666,
+      0.3011111111111111,
+      0.33555555555555555,
+      0.39222222222222225,
+      0.41888888888888887,
+      0.3888888888888889,
+      0.4111111111111111,
+      0.39666666666666667,
+      0.41,
+      0.42444444444444446
+    ]
+  },
+  "final_accuracy": 0.33625,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.65,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9404958677685951,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9473684210526315,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.62,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.946078431372549,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9585798816568047,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.9354838709677419,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9087837837837838,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.3,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9631901840490797,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.8923076923076924,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.8850574712643678,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.6945812807881774,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9401709401709402,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.2,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9834710743801653,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9173553719008265,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.9795918367346939,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.6075268817204301,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.6457399103139013,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.21,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9903846153846154,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.9622641509433962,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.675,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.50814332247557,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.2,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.39,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.4575,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.32,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.43,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.424,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.625,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9686800894854586,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.953125,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9107142857142857,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.888468809073724,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.8541666666666666,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.98,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.689119170984456,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.794392523364486,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.35,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.70703125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.7222222222222222,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.24,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.93,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.696078431372549,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.7731958762886598,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.2,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7295081967213115,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.8076923076923077,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.74,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9500831946755408,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.5,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9605734767025089,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.9655172413793104,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.875,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.8174603174603174,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.23,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9671361502347418,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9646017699115044,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9176470588235294,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.6077348066298343,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.8425925925925926,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.1,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.994413407821229,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9029126213592233,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.4966442953020134,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.5821596244131455,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.11,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.44,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.33666666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.06,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.47,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.27,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.47,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.955,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9400749063670412,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9622641509433962,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.8223234624145785,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.9024390243902439,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.2,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9833333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.5736040609137056,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.5825242718446602,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.11,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.6153846153846154,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.49673202614379086,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.13,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.6644295302013423,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.5247524752475248,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.33625,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  }
+}

add_sub_baseline_10K_1L3H510d/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb08c950f588dd85bf054c11d3921a45bb77e8a8be21e8944ddd67270255396
+size 634642298

add_sub_baseline_10K_1L3H510d/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 8e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_baseline_10K_1L3H510d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 1,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 0,
+  "dataset_size": 10000,
+  "mode": "baseline",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 158584246,
+  "run_name": "add_sub_baseline_10K_1L3H510d",
+  "git_commit": "7c8a203ce79a277a1e41a3ec0648cb73d5b2b760",
+  "timestamp": "2026-04-14T02:09:33.297198+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "sft",
+  "wandb_run_id": "6ykzfg1p",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/6ykzfg1p",
+  "final_accuracy": 0.33625,
+  "sft_accuracy": 0.33625,
+  "eval_method": "ArithmeticEvaluator"
+}