|
|
|
|
|
""" |
|
|
GAIA Benchmark Submission - Generated Space Code |
|
|
This file contains the model answers for GAIA benchmark evaluation. |
|
|
""" |
|
|
|
|
|
|
|
|
GAIA_ANSWERS = { |
|
|
"8e867cd7-cff9-4e6c-867a-ff5ddc2550be": { |
|
|
"model_answer": '3', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"a1e91b78-d3d8-4675-bb8d-62741b4b68a6": { |
|
|
"model_answer": '3', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"2d83110e-a098-4ebb-9987-066c06fa42d0": { |
|
|
"model_answer": 'right', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"cca530fc-4052-43b2-b130-b30968d8aa44": { |
|
|
"model_answer": 'Qe1+', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"4fc2f1ae-8625-45b5-ab34-ad4433bc21f8": { |
|
|
"model_answer": '• * The instructions require the answer to be "a number OR as few words as possible OR a comma separated list". Since I cannot provide the name(s), I must indicate this lack of information concisely', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"6f37996b-2ac7-44b0-8e68-6d28256631b4": { |
|
|
"model_answer": 'b, e', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"9d191bce-651d-4746-be2d-7ef8ecadb9c2": { |
|
|
"model_answer": 'Extremely', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"cabe07ed-9eca-40ea-8ead-410ef5e83f91": { |
|
|
"model_answer": '• * The question asks "What is the surname...". Since the information is not available in the provided context, I cannot provide the surname. The final answer should be "as few words as possible". "Unknown" is a single word that accurately reflects the situation where the information cannot be retrieved from the given data. It is a string, uses no articles or abbreviations, and fits the requirements', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"3cef3a44-215e-4aed-8e3b-b1e3f08063b7": { |
|
|
"model_answer": 'broccoli, celery, fresh basil, lettuce, sweet potatoes', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3": { |
|
|
"model_answer": 'Cornstarch, Granulated sugar, Lemon juice, Ripe strawberries, Salt, Unsalted butter, Vanilla extract', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"305ac316-eef6-4446-960a-92d80d542f82": { |
|
|
"model_answer": 'Wojciech', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"f918266a-b3e0-4914-865d-4faa564f1aef": { |
|
|
"model_answer": 'Python code not provided', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"3f57289b-8c60-48be-bd80-01f8099ca449": { |
|
|
"model_answer": '540', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"1f975693-876d-457b-a649-393859e79bf3": { |
|
|
"model_answer": '15, 22, 23, 24, 25, 30, 41, 42, 43', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"840bfca7-4f7b-481a-8794-c560c340185d": { |
|
|
"model_answer": '80GSFC21M0002', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"bda648d7-d618-4883-88f4-3466eabd860e": { |
|
|
"model_answer": 'Saint Petersburg', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"cf106601-ab4f-4af9-b045-5295fe67b37d": { |
|
|
"model_answer": 'CUB', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"a0c07678-e491-4bbc-8f0b-07405144218f": { |
|
|
"model_answer": 'Nagai, VerHagen', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"7bd855d8-463d-4ed5-93ca-5fe35145f733": { |
|
|
"model_answer": 'Excel file content needed', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
"5a0c1adf-205e-4841-a666-7c3ef95def9d": { |
|
|
"model_answer": 'Claus', |
|
|
"reasoning_trace": '' |
|
|
}, |
|
|
} |
|
|
|
|
|
def get_answer(task_id: str) -> dict: |
|
|
"""Get answer for a specific task ID""" |
|
|
return GAIA_ANSWERS.get(task_id, { |
|
|
"model_answer": "", |
|
|
"reasoning_trace": "Task ID not found" |
|
|
}) |
|
|
|
|
|
def get_all_answers() -> dict: |
|
|
"""Get all answers""" |
|
|
return GAIA_ANSWERS |
|
|
|
|
|
def get_task_ids() -> list: |
|
|
"""Get all task IDs""" |
|
|
return list(GAIA_ANSWERS.keys()) |
|
|
|
|
|
def get_statistics() -> dict: |
|
|
"""Get submission statistics""" |
|
|
total_tasks = len(GAIA_ANSWERS) |
|
|
answered_tasks = sum(1 for answer in GAIA_ANSWERS.values() |
|
|
if answer.get("model_answer", "").strip()) |
|
|
|
|
|
return { |
|
|
"total_tasks": total_tasks, |
|
|
"answered_tasks": answered_tasks, |
|
|
"completion_rate": answered_tasks / total_tasks if total_tasks > 0 else 0.0 |
|
|
} |
|
|
|
|
|
if __name__ == "__main__": |
|
|
|
|
|
stats = get_statistics() |
|
|
print(f"GAIA Submission Statistics:") |
|
|
print(f"Total tasks: {stats['total_tasks']}") |
|
|
print(f"Answered tasks: {stats['answered_tasks']}") |
|
|
print(f"Completion rate: {stats['completion_rate']:.2%}") |
|
|
|
