Upload add_sub_sorl_v1_abs10_K1_100K_2L1H128d/metrics.json with huggingface_hub

add_sub_sorl_v1_abs10_K1_100K_2L1H128d/metrics.json

@@ -5030,502 +5030,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.03,
-        "n_examples": 100,
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.6428571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5933884297520661,
-            "count": 605
+            "accuracy": 0.6033898305084746,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.8526315789473684,
-            "count": 95
+            "accuracy": 0.8545454545454545,
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.5542857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5833333333333334,
-            "count": 204
+            "accuracy": 0.5952380952380952,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.41420118343195267,
-            "count": 169
+            "accuracy": 0.4050632911392405,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.9032258064516129,
-            "count": 31
+            "accuracy": 0.8571428571428571,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.5202702702702703,
-            "count": 296
+            "accuracy": 0.5564516129032258,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.54,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.588957055214724,
-            "count": 163
+            "accuracy": 0.6666666666666666,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.3384615384615385,
-            "count": 130
+            "accuracy": 0.3225806451612903,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.6781609195402298,
-            "count": 87
+            "accuracy": 0.6923076923076923,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.5911330049261084,
-            "count": 203
+            "accuracy": 0.5405405405405406,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.49572649572649574,
-            "count": 117
+            "accuracy": 0.5079365079365079,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.0,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.4742857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6033057851239669,
-            "count": 121
+            "accuracy": 0.6166666666666667,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.2809917355371901,
-            "count": 121
+            "accuracy": 0.49122807017543857,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.7959183673469388,
-            "count": 49
+            "accuracy": 0.631578947368421,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.510752688172043,
-            "count": 186
+            "accuracy": 0.47115384615384615,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.4080717488789238,
-            "count": 223
+            "accuracy": 0.36363636363636365,
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.03,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.4057142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7115384615384616,
-            "count": 104
+            "accuracy": 0.6666666666666666,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.27358490566037735,
-            "count": 106
+            "accuracy": 0.23076923076923078,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.7391304347826086,
-            "count": 23
+            "accuracy": 0.8571428571428571,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.56875,
-            "count": 160
+            "accuracy": 0.5280898876404494,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.31596091205211724,
-            "count": 307
+            "accuracy": 0.2922077922077922,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.18571428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6,
-            "count": 100
+            "accuracy": 0.66,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.25,
-            "count": 100
+            "accuracy": 0.28,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.34,
-            "count": 100
+            "accuracy": 0.26,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.115,
-            "count": 400
+            "accuracy": 0.025,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.17,
-        "n_examples": 100,
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.5857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.26,
-            "count": 100
+            "accuracy": 0.22,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.84,
-            "count": 100
+            "accuracy": 0.78,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.672,
-            "count": 500
+            "accuracy": 0.62,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.02,
+        "full_accuracy": 0.015,
+        "digit_accuracy": 0.5328571428571428,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5704697986577181,
-            "count": 447
+            "accuracy": 0.5777262180974478,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.459375,
-            "count": 320
+            "accuracy": 0.4462025316455696,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.8214285714285714,
-            "count": 56
+            "accuracy": 0.8205128205128205,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.5671077504725898,
-            "count": 529
+            "accuracy": 0.5285714285714286,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.4375,
-            "count": 48
+            "accuracy": 0.5185185185185185,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.5342857142857143,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.592,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.32,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.46,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.4542857142857143,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.55,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.18,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.4457831325301205,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.17647058823529413,
+            "count": 17
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.4342857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6366666666666667,
-            "count": 300
+            "accuracy": 0.5866666666666667,
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.28,
-            "count": 100
+            "accuracy": 0.22,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.38860103626943004,
-            "count": 193
+            "accuracy": 0.42,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.205607476635514,
-            "count": 107
+            "accuracy": 0.22,
+            "count": 50
           }
         }
       },
       "add_C4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.43714285714285717,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.68,
-            "count": 200
+            "accuracy": 0.7,
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.27,
-            "count": 100
+            "accuracy": 0.34,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.33203125,
-            "count": 256
+            "accuracy": 0.3181818181818182,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.2916666666666667,
-            "count": 144
+            "accuracy": 0.35294117647058826,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.0,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.3485714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.63,
-            "count": 100
+            "accuracy": 0.62,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.21,
-            "count": 100
+            "accuracy": 0.28,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.4150326797385621,
-            "count": 306
+            "accuracy": 0.3972602739726027,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.2268041237113402,
-            "count": 194
+            "accuracy": 0.18269230769230768,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.4857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.3,
-            "count": 100
+            "accuracy": 0.24,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.44808743169398907,
-            "count": 366
+            "accuracy": 0.49206349206349204,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.5854700854700855,
-            "count": 234
+            "accuracy": 0.5855855855855856,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.21,
-        "n_examples": 100,
+        "full_accuracy": 0.16,
+        "digit_accuracy": 0.7828571428571428,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7903494176372712,
-            "count": 601
+            "accuracy": 0.7623762376237624,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.9292929292929293,
-            "count": 99
+            "accuracy": 0.9148936170212766,
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.04,
-        "n_examples": 100,
+        "full_accuracy": 0.1,
+        "digit_accuracy": 0.74,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8172043010752689,
-            "count": 279
+            "accuracy": 0.8439716312056738,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.6758620689655173,
-            "count": 145
+            "accuracy": 0.7222222222222222,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.9583333333333334,
-            "count": 24
+            "accuracy": 0.7777777777777778,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.5753968253968254,
-            "count": 252
+            "accuracy": 0.6218487394957983,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.04,
-        "n_examples": 100,
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.6457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.863849765258216,
-            "count": 213
+            "accuracy": 0.8482142857142857,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.672566371681416,
-            "count": 113
+            "accuracy": 0.5094339622641509,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9411764705882353,
-            "count": 85
+            "accuracy": 0.851063829787234,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.4696132596685083,
-            "count": 181
+            "accuracy": 0.4823529411764706,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.4444444444444444,
-            "count": 108
+            "accuracy": 0.4339622641509434,
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.04,
-        "n_examples": 100,
+        "digit_accuracy": 0.5914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9162011173184358,
-            "count": 179
+            "accuracy": 0.8762886597938144,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.5922330097087378,
-            "count": 103
+            "accuracy": 0.6862745098039216,
+            "count": 51
           },
           "ME": {
-            "accuracy": 0.9285714285714286,
-            "count": 56
+            "accuracy": 0.9629629629629629,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.5033557046979866,
-            "count": 149
+            "accuracy": 0.5,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.22065727699530516,
-            "count": 213
+            "accuracy": 0.2376237623762376,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.4514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.79,
-            "count": 200
+            "accuracy": 0.73,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.44,
-            "count": 100
+            "accuracy": 0.6,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.57,
-            "count": 100
+            "accuracy": 0.54,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.11,
-            "count": 300
+            "accuracy": 0.18666666666666668,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.05,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.3514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.42,
-            "count": 100
+            "accuracy": 0.46,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.61,
-            "count": 100
+            "accuracy": 0.54,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.11,
-            "count": 400
+            "accuracy": 0.115,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.095,
+        "full_accuracy": 0.055,
+        "digit_accuracy": 0.6971428571428572,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7883333333333333,
-            "count": 600
+            "accuracy": 0.8105263157894737,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.6853932584269663,
-            "count": 267
+            "accuracy": 0.6462093862815884,
+            "count": 277
           },
           "ME": {
-            "accuracy": 0.9433962264150944,
+            "accuracy": 0.9245283018867925,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.6264236902050114,
-            "count": 439
+            "accuracy": 0.583864118895966,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.43902439024390244,
-            "count": 41
+            "accuracy": 0.3793103448275862,
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.02,
-        "n_examples": 100,
+        "digit_accuracy": 0.5914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7133333333333334,
-            "count": 300
+            "accuracy": 0.7333333333333333,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.67,
-            "count": 100
+            "accuracy": 0.64,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.5279187817258884,
-            "count": 197
+            "accuracy": 0.45544554455445546,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.3300970873786408,
-            "count": 103
+            "accuracy": 0.3877551020408163,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.52,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.765,
-            "count": 200
+            "accuracy": 0.77,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.59,
-            "count": 100
+            "accuracy": 0.66,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.38866396761133604,
-            "count": 247
+            "accuracy": 0.4297520661157025,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.3333333333333333,
-            "count": 153
+            "accuracy": 0.25316455696202533,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.0,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.5314285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.53,
-            "count": 100
+            "accuracy": 0.56,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.46308724832214765,
-            "count": 298
+            "accuracy": 0.48026315789473684,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.31683168316831684,
-            "count": 202
+            "accuracy": 0.35714285714285715,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.03791666666666667,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.032,
+      "digit_accuracy": 0.54,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -5534,502 +5599,567 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.98,
-        "n_examples": 100,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.996694214876033,
-            "count": 605
+            "accuracy": 0.9966101694915255,
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.93,
-        "n_examples": 100,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9852941176470589,
-            "count": 204
+            "accuracy": 1.0,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9822485207100592,
-            "count": 169
+            "accuracy": 1.0,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.967741935483871,
-            "count": 31
+            "accuracy": 1.0,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.9932432432432432,
-            "count": 296
+            "accuracy": 0.9919354838709677,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.81,
-        "n_examples": 100,
+        "full_accuracy": 0.66,
+        "digit_accuracy": 0.9514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9877300613496932,
-            "count": 163
+            "accuracy": 1.0,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.9923076923076923,
-            "count": 130
+            "accuracy": 0.967741935483871,
+            "count": 62
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 87
+            "accuracy": 0.9743589743589743,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.916256157635468,
-            "count": 203
+            "accuracy": 0.8738738738738738,
+            "count": 111
           },
           "US": {
             "accuracy": 1.0,
-            "count": 117
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.63,
-        "n_examples": 100,
+        "full_accuracy": 0.7,
+        "digit_accuracy": 0.9514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.9752066115702479,
-            "count": 121
+            "accuracy": 0.9649122807017544,
+            "count": 57
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 49
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.8118279569892473,
-            "count": 186
+            "accuracy": 0.8557692307692307,
+            "count": 104
           },
           "US": {
             "accuracy": 1.0,
-            "count": 223
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.59,
-        "n_examples": 100,
+        "full_accuracy": 0.76,
+        "digit_accuracy": 0.9542857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 48
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 106
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.75,
-            "count": 160
+            "accuracy": 0.8651685393258427,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.9739413680781759,
-            "count": 307
+            "accuracy": 0.974025974025974,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.34,
-        "n_examples": 100,
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.7742857142857142,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.39,
-            "count": 100
+            "accuracy": 0.22,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.85,
-            "count": 400
+            "accuracy": 0.8,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.45,
-        "n_examples": 100,
+        "full_accuracy": 0.42,
+        "digit_accuracy": 0.7914285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.53,
-            "count": 100
+            "accuracy": 0.54,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.82,
-            "count": 500
+            "accuracy": 0.8,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.945,
+        "full_accuracy": 0.935,
+        "digit_accuracy": 0.9907142857142858,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9932885906040269,
-            "count": 447
+            "accuracy": 0.9976798143851509,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.996875,
-            "count": 320
+            "accuracy": 1.0,
+            "count": 316
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9867674858223062,
-            "count": 529
+            "accuracy": 0.9785714285714285,
+            "count": 560
           },
           "US": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.82,
+        "digit_accuracy": 0.9742857142857143,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9036144578313253,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.78,
-        "n_examples": 100,
+        "full_accuracy": 0.64,
+        "digit_accuracy": 0.9485714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8860103626943006,
-            "count": 193
+            "accuracy": 0.82,
+            "count": 100
           },
           "US": {
             "accuracy": 1.0,
-            "count": 107
+            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.77,
-        "n_examples": 100,
+        "full_accuracy": 0.84,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.90625,
-            "count": 256
+            "accuracy": 0.9393939393939394,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.9722222222222222,
-            "count": 144
+            "accuracy": 1.0,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.75,
-        "n_examples": 100,
+        "full_accuracy": 0.7,
+        "digit_accuracy": 0.9457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.9281045751633987,
-            "count": 306
+            "accuracy": 0.8972602739726028,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.9587628865979382,
-            "count": 194
+            "accuracy": 0.9615384615384616,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.69,
-        "n_examples": 100,
+        "full_accuracy": 0.8,
+        "digit_accuracy": 0.9628571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.912568306010929,
-            "count": 366
+            "accuracy": 0.9417989417989417,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.9829059829059829,
-            "count": 234
+            "accuracy": 0.9819819819819819,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.87,
-        "n_examples": 100,
+        "full_accuracy": 0.86,
+        "digit_accuracy": 0.98,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.978369384359401,
-            "count": 601
+            "accuracy": 0.976897689768977,
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.88,
-        "n_examples": 100,
+        "full_accuracy": 0.84,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.982078853046595,
-            "count": 279
+            "accuracy": 0.9645390070921985,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.993103448275862,
-            "count": 145
+            "accuracy": 0.9861111111111112,
+            "count": 72
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 24
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.9761904761904762,
-            "count": 252
+            "accuracy": 0.9831932773109243,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.68,
-        "n_examples": 100,
+        "full_accuracy": 0.66,
+        "digit_accuracy": 0.9485714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9859154929577465,
-            "count": 213
+            "accuracy": 1.0,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.9823008849557522,
-            "count": 113
+            "accuracy": 0.9811320754716981,
+            "count": 53
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 85
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.8342541436464088,
-            "count": 181
+            "accuracy": 0.8,
+            "count": 85
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 108
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.5,
-        "n_examples": 100,
+        "full_accuracy": 0.46,
+        "digit_accuracy": 0.9142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9888268156424581,
-            "count": 179
+            "accuracy": 1.0,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.9514563106796117,
-            "count": 103
+            "accuracy": 0.9411764705882353,
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.6778523489932886,
-            "count": 149
+            "accuracy": 0.6351351351351351,
+            "count": 74
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 213
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.33,
-        "n_examples": 100,
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.8942857142857142,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.43,
-            "count": 100
+            "accuracy": 0.4,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.9333333333333333,
-            "count": 300
+            "accuracy": 0.9533333333333334,
+            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.08,
-        "n_examples": 100,
+        "digit_accuracy": 0.7657142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.36,
-            "count": 100
+            "accuracy": 0.5,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.725,
-            "count": 400
+            "accuracy": 0.715,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.85,
+        "full_accuracy": 0.82,
+        "digit_accuracy": 0.9735714285714285,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.99,
-            "count": 600
+            "accuracy": 0.987719298245614,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9812734082397003,
-            "count": 267
+            "accuracy": 0.9711191335740073,
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9567198177676538,
-            "count": 439
+            "accuracy": 0.9532908704883227,
+            "count": 471
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 41
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.69,
-        "n_examples": 100,
+        "full_accuracy": 0.74,
+        "digit_accuracy": 0.9628571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9833333333333333,
-            "count": 300
+            "accuracy": 0.9933333333333333,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8730964467005076,
-            "count": 197
+            "accuracy": 0.900990099009901,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.9805825242718447,
-            "count": 103
+            "accuracy": 0.9795918367346939,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.69,
-        "n_examples": 100,
+        "full_accuracy": 0.64,
+        "digit_accuracy": 0.9428571428571428,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8785425101214575,
-            "count": 247
+            "accuracy": 0.8347107438016529,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.9738562091503268,
-            "count": 153
+            "accuracy": 1.0,
+            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.46,
-        "n_examples": 100,
+        "digit_accuracy": 0.9114285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8288590604026845,
-            "count": 298
+            "accuracy": 0.8223684210526315,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.9603960396039604,
-            "count": 202
+            "accuracy": 0.9591836734693877,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.6870833333333334,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.7193333333333334,
+      "digit_accuracy": 0.9459047619047619,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.6870833333333334,