Upload script_1/flexitok--bpe_script_Arab_16000_overlap.json with huggingface_hub

script_1/flexitok--bpe_script_Arab_16000_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.8657147914331229, "expected_training_ratio_in_superset": 0.07870134467573844, "num_tokens": 14673}, "2": {"ratio_to_total_tokens": 0.00430703876334887, "expected_training_ratio_in_superset": 0.0007830979569725219, "num_tokens": 73}, "3": {"ratio_to_total_tokens": 0.002419021771195941, "expected_training_ratio_in_superset": 0.0006597332103261657, "num_tokens": 41}, "4": {"ratio_to_total_tokens": 0.0018880169921529294, "expected_training_ratio_in_superset": 0.0006865516335101562, "num_tokens": 32}, "5": {"ratio_to_total_tokens": 0.003540031860286743, "expected_training_ratio_in_superset": 0.0016091053910394285, "num_tokens": 60}, "6": {"ratio_to_total_tokens": 0.004189037701339312, "expected_training_ratio_in_superset": 0.002284929655275988, "num_tokens": 71}, "7": {"ratio_to_total_tokens": 0.005369048321434893, "expected_training_ratio_in_superset": 0.003416667113640386, "num_tokens": 91}, "8": {"ratio_to_total_tokens": 0.005841052569473125, "expected_training_ratio_in_superset": 0.004248038232344091, "num_tokens": 99}, "9": {"ratio_to_total_tokens": 0.005841052569473125, "expected_training_ratio_in_superset": 0.004779043011387102, "num_tokens": 99}, "10": {"ratio_to_total_tokens": 0.006608059472535253, "expected_training_ratio_in_superset": 0.006007326793213866, "num_tokens": 112}, "11": {"ratio_to_total_tokens": 0.09428284854563691, "expected_training_ratio_in_superset": 0.09428284854563691, "num_tokens": 1598}, "total_training_compared_to_full_model": 0.19745868621908505}