{
  "config_general": {
    "lighteval_sha": "?",
    "num_fewshot_seeds": 1,
    "override_batch_size": 1,
    "max_samples": null,
    "job_id": "",
    "start_time": 549406.183073153,
    "end_time": 549696.457783003,
    "total_evaluation_time_secondes": "290.2747098499676",
    "model_name": "HuggingFaceH4/zephyr-7b-gemma-v0.1",
    "model_sha": "03b3427d0ed07d2e0f86c0a7e53d82d4beef9540",
    "model_dtype": "torch.bfloat16",
    "model_size": "15.9 GB",
    "config": null
  },
  "results": {
    "lighteval|agieval:aqua-rat|0": {
      "acc": 0.2125984251968504,
      "acc_stderr": 0.025722779833723054,
      "acc_norm": 0.2125984251968504,
      "acc_norm_stderr": 0.02572277983372305
    },
    "lighteval|agieval:gaokao-biology|0": {
      "acc": 0.35714285714285715,
      "acc_stderr": 0.033144012047664914,
      "acc_norm": 0.38095238095238093,
      "acc_norm_stderr": 0.03359110046749989
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "acc": 0.26570048309178745,
      "acc_stderr": 0.030775079470103068,
      "acc_norm": 0.2946859903381642,
      "acc_norm_stderr": 0.03176416108295297
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "acc": 0.2967479674796748,
      "acc_stderr": 0.029185445861037912,
      "acc_norm": 0.3008130081300813,
      "acc_norm_stderr": 0.02929961637067325
    },
    "lighteval|agieval:gaokao-english|0": {
      "acc": 0.6111111111111112,
      "acc_stderr": 0.027914055510468008,
      "acc_norm": 0.5620915032679739,
      "acc_norm_stderr": 0.028408302020332687
    },
    "lighteval|agieval:gaokao-geography|0": {
      "acc": 0.46733668341708545,
      "acc_stderr": 0.03545755092964412,
      "acc_norm": 0.457286432160804,
      "acc_norm_stderr": 0.035403557368657
    },
    "lighteval|agieval:gaokao-history|0": {
      "acc": 0.451063829787234,
      "acc_stderr": 0.032529096196131965,
      "acc_norm": 0.4127659574468085,
      "acc_norm_stderr": 0.03218471141400352
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "acc": 0.27635327635327633,
      "acc_stderr": 0.023903505003127223,
      "acc_norm": 0.27635327635327633,
      "acc_norm_stderr": 0.023903505003127216
    },
    "lighteval|agieval:gaokao-physics|0": {
      "acc": 0.36,
      "acc_stderr": 0.03402629784040017,
      "acc_norm": 0.37,
      "acc_norm_stderr": 0.0342250899767933
    },
    "lighteval|agieval:logiqa-en|0": {
      "acc": 0.35944700460829493,
      "acc_stderr": 0.018820809084481267,
      "acc_norm": 0.3486943164362519,
      "acc_norm_stderr": 0.018692104055797923
    },
    "lighteval|agieval:logiqa-zh|0": {
      "acc": 0.31336405529953915,
      "acc_stderr": 0.01819412517802074,
      "acc_norm": 0.36251920122887865,
      "acc_norm_stderr": 0.018855687979585072
    },
    "lighteval|agieval:lsat-ar|0": {
      "acc": 0.2217391304347826,
      "acc_stderr": 0.027451496604058913,
      "acc_norm": 0.16521739130434782,
      "acc_norm_stderr": 0.02454125880854541
    },
    "lighteval|agieval:lsat-lr|0": {
      "acc": 0.3627450980392157,
      "acc_stderr": 0.021310737393780418,
      "acc_norm": 0.3215686274509804,
      "acc_norm_stderr": 0.020702886736741092
    },
    "lighteval|agieval:lsat-rc|0": {
      "acc": 0.5055762081784386,
      "acc_stderr": 0.03054046165569704,
      "acc_norm": 0.3754646840148699,
      "acc_norm_stderr": 0.029579828435446678
    },
    "lighteval|agieval:sat-en|0": {
      "acc": 0.6747572815533981,
      "acc_stderr": 0.03271904737596389,
      "acc_norm": 0.49514563106796117,
      "acc_norm_stderr": 0.03491986890584391
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "acc": 0.46601941747572817,
      "acc_stderr": 0.03484077510347999,
      "acc_norm": 0.3446601941747573,
      "acc_norm_stderr": 0.03319341285859081
    },
    "lighteval|agieval:sat-math|0": {
      "acc": 0.42272727272727273,
      "acc_stderr": 0.03338094264093533,
      "acc_norm": 0.3409090909090909,
      "acc_norm_stderr": 0.03203095553573995
    },
    "lighteval|agieval:_average|0": {
      "acc": 0.38967235893509095,
      "acc_stderr": 0.028818601042865762,
      "acc_norm": 0.35421918296667515,
      "acc_norm_stderr": 0.028648166285532566
    },
    "all": {
      "acc": 0.38967235893509095,
      "acc_stderr": 0.028818601042865762,
      "acc_norm": 0.35421918296667515,
      "acc_norm_stderr": 0.028648166285532566
    }
  },
  "versions": {
    "lighteval|agieval:aqua-rat|0": 0,
    "lighteval|agieval:gaokao-biology|0": 0,
    "lighteval|agieval:gaokao-chemistry|0": 0,
    "lighteval|agieval:gaokao-chinese|0": 0,
    "lighteval|agieval:gaokao-english|0": 0,
    "lighteval|agieval:gaokao-geography|0": 0,
    "lighteval|agieval:gaokao-history|0": 0,
    "lighteval|agieval:gaokao-mathqa|0": 0,
    "lighteval|agieval:gaokao-physics|0": 0,
    "lighteval|agieval:logiqa-en|0": 0,
    "lighteval|agieval:logiqa-zh|0": 0,
    "lighteval|agieval:lsat-ar|0": 0,
    "lighteval|agieval:lsat-lr|0": 0,
    "lighteval|agieval:lsat-rc|0": 0,
    "lighteval|agieval:sat-en|0": 0,
    "lighteval|agieval:sat-en-without-passage|0": 0,
    "lighteval|agieval:sat-math|0": 0
  },
  "config_tasks": {
    "lighteval|agieval:aqua-rat": {
      "name": "agieval:aqua-rat",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-aqua-rat",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 254,
      "effective_num_docs": 254,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-biology": {
      "name": "agieval:gaokao-biology",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-biology",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 210,
      "effective_num_docs": 210,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chemistry": {
      "name": "agieval:gaokao-chemistry",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chemistry",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 207,
      "effective_num_docs": 207,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chinese": {
      "name": "agieval:gaokao-chinese",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chinese",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 246,
      "effective_num_docs": 246,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-english": {
      "name": "agieval:gaokao-english",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-english",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 306,
      "effective_num_docs": 306,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-geography": {
      "name": "agieval:gaokao-geography",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-geography",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 199,
      "effective_num_docs": 199,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-history": {
      "name": "agieval:gaokao-history",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-history",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 235,
      "effective_num_docs": 235,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-mathqa": {
      "name": "agieval:gaokao-mathqa",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-mathqa",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 351,
      "effective_num_docs": 351,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-physics": {
      "name": "agieval:gaokao-physics",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-physics",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 200,
      "effective_num_docs": 200,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-en": {
      "name": "agieval:logiqa-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-zh": {
      "name": "agieval:logiqa-zh",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-zh",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-ar": {
      "name": "agieval:lsat-ar",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-ar",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 230,
      "effective_num_docs": 230,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-lr": {
      "name": "agieval:lsat-lr",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-lr",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 510,
      "effective_num_docs": 510,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-rc": {
      "name": "agieval:lsat-rc",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-rc",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 269,
      "effective_num_docs": 269,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en": {
      "name": "agieval:sat-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en-without-passage": {
      "name": "agieval:sat-en-without-passage",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en-without-passage",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-math": {
      "name": "agieval:sat-math",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-math",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 220,
      "effective_num_docs": 220,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    }
  },
  "summary_tasks": {
    "lighteval|agieval:aqua-rat|0": {
      "hashes": {
        "hash_examples": "f09607f69e5b7525",
        "hash_full_prompts": "8ad711fb8fb77d94",
        "hash_input_tokens": "3ca91e227e1f5ee8",
        "hash_cont_tokens": "68076809549d1d3f"
      },
      "truncated": 0,
      "non_truncated": 254,
      "padded": 1270,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-biology|0": {
      "hashes": {
        "hash_examples": "f262eaf4a72db963",
        "hash_full_prompts": "33999b9b989424c2",
        "hash_input_tokens": "8bf36c825bbcbace",
        "hash_cont_tokens": "6e251426994bce31"
      },
      "truncated": 0,
      "non_truncated": 210,
      "padded": 833,
      "non_padded": 7,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "hashes": {
        "hash_examples": "47f2e649f58d9da5",
        "hash_full_prompts": "7d7d09f9eb879955",
        "hash_input_tokens": "8f96a029d955595e",
        "hash_cont_tokens": "4871f0178c1adc9e"
      },
      "truncated": 0,
      "non_truncated": 207,
      "padded": 827,
      "non_padded": 4,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "hashes": {
        "hash_examples": "1010b21fde4726ab",
        "hash_full_prompts": "60c65321d5cc8691",
        "hash_input_tokens": "e71306cf9d3a3034",
        "hash_cont_tokens": "0be39ceaef751ad1"
      },
      "truncated": 0,
      "non_truncated": 246,
      "padded": 979,
      "non_padded": 5,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-english|0": {
      "hashes": {
        "hash_examples": "4864e492a350ae93",
        "hash_full_prompts": "65c242dcefa907ed",
        "hash_input_tokens": "f8bfdf5178b55ada",
        "hash_cont_tokens": "689c6ea1272771f8"
      },
      "truncated": 0,
      "non_truncated": 306,
      "padded": 1224,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-geography|0": {
      "hashes": {
        "hash_examples": "ec3a021e37650e7d",
        "hash_full_prompts": "d66698e42924982f",
        "hash_input_tokens": "4df6fd6f05dd97b2",
        "hash_cont_tokens": "fc7dd10486347853"
      },
      "truncated": 0,
      "non_truncated": 199,
      "padded": 793,
      "non_padded": 3,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-history|0": {
      "hashes": {
        "hash_examples": "b3fad1596f1ae1f9",
        "hash_full_prompts": "b688278c6bb839a5",
        "hash_input_tokens": "bf01c4ae7bccfb50",
        "hash_cont_tokens": "94a46bd09bc7f9a7"
      },
      "truncated": 0,
      "non_truncated": 235,
      "padded": 934,
      "non_padded": 6,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "hashes": {
        "hash_examples": "1d1088556861b0b0",
        "hash_full_prompts": "a5f205e77d0a99c2",
        "hash_input_tokens": "e955f3458ac121d3",
        "hash_cont_tokens": "d19aa287b771e823"
      },
      "truncated": 0,
      "non_truncated": 351,
      "padded": 1396,
      "non_padded": 8,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-physics|0": {
      "hashes": {
        "hash_examples": "eb05f035c7bfca2f",
        "hash_full_prompts": "408cff052da9e941",
        "hash_input_tokens": "240bc1cf51689ef3",
        "hash_cont_tokens": "78f68d934030f9d6"
      },
      "truncated": 0,
      "non_truncated": 200,
      "padded": 792,
      "non_padded": 8,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-en|0": {
      "hashes": {
        "hash_examples": "0a688a45f69c21e0",
        "hash_full_prompts": "4abde5e3d4c7b3f8",
        "hash_input_tokens": "b49fe92bef71b0db",
        "hash_cont_tokens": "78b51c7dda5a457a"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2592,
      "non_padded": 12,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-zh|0": {
      "hashes": {
        "hash_examples": "620d6888b6012ea5",
        "hash_full_prompts": "dc4ddbb9c38aef67",
        "hash_input_tokens": "2eef4f372fdad7b7",
        "hash_cont_tokens": "33ab0a1feb3a2fed"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2588,
      "non_padded": 16,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-ar|0": {
      "hashes": {
        "hash_examples": "627c8f5ccd5da209",
        "hash_full_prompts": "85f4e2da30c42407",
        "hash_input_tokens": "b8ef0c74c45ff792",
        "hash_cont_tokens": "5a4d3fed21889b2c"
      },
      "truncated": 0,
      "non_truncated": 230,
      "padded": 1145,
      "non_padded": 5,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-lr|0": {
      "hashes": {
        "hash_examples": "794641c86de172f5",
        "hash_full_prompts": "833dd104fba3a50a",
        "hash_input_tokens": "8bcb7119e593166c",
        "hash_cont_tokens": "449ff43e1d759ce9"
      },
      "truncated": 0,
      "non_truncated": 510,
      "padded": 2537,
      "non_padded": 13,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-rc|0": {
      "hashes": {
        "hash_examples": "35981ed917ea01cf",
        "hash_full_prompts": "1f6ee9e7b383dee6",
        "hash_input_tokens": "0e5dcc5d9ba3acb4",
        "hash_cont_tokens": "3c2d8acf3e02c384"
      },
      "truncated": 0,
      "non_truncated": 269,
      "padded": 1345,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en|0": {
      "hashes": {
        "hash_examples": "041c39c646536a1e",
        "hash_full_prompts": "dc367ba3deb69c80",
        "hash_input_tokens": "8f6bcc50dd80c45f",
        "hash_cont_tokens": "a6f1ab815d02f06d"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 821,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "hashes": {
        "hash_examples": "e4d9284367dff68f",
        "hash_full_prompts": "dc84a50c7cf1b0da",
        "hash_input_tokens": "c63a20ff70e10200",
        "hash_cont_tokens": "a6f1ab815d02f06d"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 811,
      "non_padded": 10,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-math|0": {
      "hashes": {
        "hash_examples": "01db7291603fc1a0",
        "hash_full_prompts": "fd1ffdeea790b637",
        "hash_input_tokens": "7d57211e46d19552",
        "hash_cont_tokens": "51a49f5633293d60"
      },
      "truncated": 0,
      "non_truncated": 220,
      "padded": 875,
      "non_padded": 5,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    }
  },
  "summary_general": {
    "hashes": {
      "hash_examples": "da3af66181f18ddf",
      "hash_full_prompts": "6cd70cd84b2d90cb",
      "hash_input_tokens": "2c432ff3076aeb5a",
      "hash_cont_tokens": "a3082c4ed8f63f0b"
    },
    "truncated": 0,
    "non_truncated": 5151,
    "padded": 21762,
    "non_padded": 102,
    "num_truncated_few_shots": 0
  }
}