rabbits-leaderboard / data /raw-eval-outputs /Qwen-Qwen2-72B_results.json
magilogi
rabbits-leaderboard-v0.1
4c59875
raw
history blame
6.88 kB
{
"results": {
"b4b": {
"acc,none": 0.8372379778051788,
"acc_stderr,none": 0.07216098703042964,
"acc_norm,none": 0.8372379778051788,
"acc_norm_stderr,none": 0.07216098703042964,
"alias": "b4b"
},
"b4bqa": {
"acc,none": 0.91015625,
"acc_stderr,none": 0.006757003132881115,
"acc_norm,none": 0.91015625,
"acc_norm_stderr,none": 0.006757003132881115,
"alias": " - b4bqa"
},
"medmcqa_g2b": {
"acc,none": 0.7155172413793104,
"acc_stderr,none": 0.024219952635630794,
"acc_norm,none": 0.7155172413793104,
"acc_norm_stderr,none": 0.024219952635630794,
"alias": " - medmcqa_g2b"
},
"medmcqa_orig_filtered": {
"acc,none": 0.7787356321839081,
"acc_stderr,none": 0.02228363451068677,
"acc_norm,none": 0.7787356321839081,
"acc_norm_stderr,none": 0.02228363451068677,
"alias": " - medmcqa_orig_filtered"
},
"medqa_4options_g2b": {
"acc,none": 0.7407407407407407,
"acc_stderr,none": 0.022569897074918417,
"acc_norm,none": 0.7407407407407407,
"acc_norm_stderr,none": 0.022569897074918417,
"alias": " - medqa_4options_g2b"
},
"medqa_4options_orig_filtered": {
"acc,none": 0.753968253968254,
"acc_stderr,none": 0.022182037202948368,
"acc_norm,none": 0.753968253968254,
"acc_norm_stderr,none": 0.022182037202948368,
"alias": " - medqa_4options_orig_filtered"
}
},
"groups": {
"b4b": {
"acc,none": 0.8372379778051788,
"acc_stderr,none": 0.07216098703042964,
"acc_norm,none": 0.8372379778051788,
"acc_norm_stderr,none": 0.07216098703042964,
"alias": "b4b"
}
},
"configs": {
"b4bqa": {
"task": "b4bqa",
"dataset_path": "AIM-Harvard/b4b_drug_qa",
"test_split": "test",
"doc_to_text": "<function process_cd at 0x7fe2a537cf70>",
"doc_to_target": "correct_choice",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false
},
"medmcqa_g2b": {
"task": "medmcqa_g2b",
"dataset_path": "AIM-Harvard/medmcqa_generic_to_brand",
"training_split": "train",
"validation_split": "validation",
"test_split": "validation",
"doc_to_text": "<function doc_to_text at 0x7fe2a4fa0430>",
"doc_to_target": "cop",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "{{question}}"
},
"medmcqa_orig_filtered": {
"task": "medmcqa_orig_filtered",
"dataset_path": "AIM-Harvard/medmcqa_original",
"training_split": "train",
"validation_split": "validation",
"test_split": "validation",
"doc_to_text": "<function doc_to_text at 0x7fe2a4fa05e0>",
"doc_to_target": "cop",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": true,
"doc_to_decontamination_query": "{{question}}"
},
"medqa_4options_g2b": {
"task": "medqa_4options_g2b",
"dataset_path": "AIM-Harvard/gbaker_medqa_usmle_4_options_hf_generic_to_brand",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "<function doc_to_text at 0x7fe2a4fa35b0>",
"doc_to_target": "<function doc_to_target at 0x7fe2a4fa3910>",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false
},
"medqa_4options_orig_filtered": {
"task": "medqa_4options_orig_filtered",
"dataset_path": "AIM-Harvard/gbaker_medqa_usmle_4_options_hf_original",
"training_split": "train",
"validation_split": "validation",
"test_split": "test",
"doc_to_text": "<function doc_to_text at 0x7fe2a4fa3b50>",
"doc_to_target": "<function doc_to_target at 0x7fe2a4fa3d90>",
"doc_to_choice": [
"A",
"B",
"C",
"D"
],
"description": "",
"target_delimiter": " ",
"fewshot_delimiter": "\n\n",
"metric_list": [
{
"metric": "acc",
"aggregation": "mean",
"higher_is_better": true
},
{
"metric": "acc_norm",
"aggregation": "mean",
"higher_is_better": true
}
],
"output_type": "multiple_choice",
"repeats": 1,
"should_decontaminate": false
}
},
"versions": {
"b4b": "N/A",
"b4bqa": "Yaml",
"medmcqa_g2b": "Yaml",
"medmcqa_orig_filtered": "Yaml",
"medqa_4options_g2b": "Yaml",
"medqa_4options_orig_filtered": "Yaml"
},
"n-shot": {
"b4b": 0,
"b4bqa": 0,
"medmcqa_g2b": 0,
"medmcqa_orig_filtered": 0,
"medqa_4options_g2b": 0,
"medqa_4options_orig_filtered": 0
},
"config": {
"model": "hf",
"model_args": "pretrained=Qwen/Qwen2-72B,load_in_4bit=True",
"batch_size": "4",
"batch_sizes": [],
"device": "cuda:0",
"use_cache": null,
"limit": null,
"bootstrap_iters": 100000,
"gen_kwargs": null
},
"git_hash": "928c7657"
}