rabbits-leaderboard / data /raw-eval-outputs /meta-llama-Meta-Llama-3-8B_results.json
magilogi
rabbits-leaderboard-v0.1
4c59875
raw
history blame
6.91 kB
{
"results": {
"b4b": {
"acc,none": 0.7120838471023427,
"acc_stderr,none": 0.11202233860795015,
"acc_norm,none": 0.7120838471023427,
"acc_norm_stderr,none": 0.11202233860795015,
"alias": "b4b"
},
"b4bqa": {
"acc,none": 0.8270089285714286,
"acc_stderr,none": 0.00893756370730241,
"acc_norm,none": 0.8270089285714286,
"acc_norm_stderr,none": 0.00893756370730241,
"alias": " - b4bqa"
},
"medmcqa_g2b": {
"acc,none": 0.5287356321839081,
"acc_stderr,none": 0.02679704183010415,
"acc_norm,none": 0.5287356321839081,
"acc_norm_stderr,none": 0.02679704183010415,
"alias": " - medmcqa_g2b"
},
"medmcqa_orig_filtered": {
"acc,none": 0.5919540229885057,
"acc_stderr,none": 0.026383584629731508,
"acc_norm,none": 0.5919540229885057,
"acc_norm_stderr,none": 0.026383584629731508,
"alias": " - medmcqa_orig_filtered"
},
"medqa_4options_g2b": {
"acc,none": 0.5502645502645502,
"acc_stderr,none": 0.025620857042936655,
"acc_norm,none": 0.5502645502645502,
"acc_norm_stderr,none": 0.025620857042936655,
"alias": " - medqa_4options_g2b"
},
"medqa_4options_orig_filtered": {
"acc,none": 0.6084656084656085,
"acc_stderr,none": 0.025138091388851102,
"acc_norm,none": 0.6084656084656085,
"acc_norm_stderr,none": 0.025138091388851102,
"alias": " - medqa_4options_orig_filtered"
}
},
"groups": {
"b4b": {
"acc,none": 0.7120838471023427,
"acc_stderr,none": 0.11202233860795015,
"acc_norm,none": 0.7120838471023427,
"acc_norm_stderr,none": 0.11202233860795015,
"alias": "b4b"
}
},
"configs": {
"b4bqa": {
"task": "b4bqa",
"dataset_path": "AIM-Harvard/b4b_drug_qa",
"test_split": "test",
"doc_to_text": "<function process_cd at 0x7ff55118d090>",
"doc_to_target": "correct_choice",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false
},
"medmcqa_g2b": {
"task": "medmcqa_g2b",
"dataset_path": "AIM-Harvard/medmcqa_generic_to_brand",
"training_split": "train",
"validation_split": "validation",
"test_split": "validation",
"doc_to_text": "<function doc_to_text at 0x7ff55058c550>",
"doc_to_target": "cop",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "{{question}}"
},
"medmcqa_orig_filtered": {
"task": "medmcqa_orig_filtered",
"dataset_path": "AIM-Harvard/medmcqa_original",
"training_split": "train",
"validation_split": "validation",
"test_split": "validation",
"doc_to_text": "<function doc_to_text at 0x7ff55058c700>",
"doc_to_target": "cop",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "{{question}}"
},
"medqa_4options_g2b": {
"task": "medqa_4options_g2b",
"dataset_path": "AIM-Harvard/gbaker_medqa_usmle_4_options_hf_generic_to_brand",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "<function doc_to_text at 0x7ff55058f6d0>",
"doc_to_target": "<function doc_to_target at 0x7ff55058fa30>",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false
},
"medqa_4options_orig_filtered": {
"task": "medqa_4options_orig_filtered",
"dataset_path": "AIM-Harvard/gbaker_medqa_usmle_4_options_hf_original",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "<function doc_to_text at 0x7ff55058fc70>",
"doc_to_target": "<function doc_to_target at 0x7ff55058feb0>",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false
}
},
"versions": {
"b4b": "N/A",
"b4bqa": "Yaml",
"medmcqa_g2b": "Yaml",
"medmcqa_orig_filtered": "Yaml",
"medqa_4options_g2b": "Yaml",
"medqa_4options_orig_filtered": "Yaml"
},
"n-shot": {
"b4b": 0,
"b4bqa": 0,
"medmcqa_g2b": 0,
"medmcqa_orig_filtered": 0,
"medqa_4options_g2b": 0,
"medqa_4options_orig_filtered": 0
},
"config": {
"model": "hf",
"model_args": "pretrained=meta-llama/Meta-Llama-3-8B,load_in_4bit=True",
"batch_size": "4",
"batch_sizes": [],
"device": "cuda:0",
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null
},
"git_hash": "928c7657"
}